diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index a404d5e43..e03d8f9cd 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -7,12 +7,12 @@ defmodule Plausible.ClickhouseEventV2 do @primary_key false schema "events_v2" do - field :name, :string - field :site_id, Ch.Types.UInt64 + field :name, Ch, type: "LowCardinality(String)" + field :site_id, Ch, type: "UInt64" field :hostname, :string field :pathname, :string - field :user_id, Ch.Types.UInt64 - field :session_id, Ch.Types.UInt64 + field :user_id, Ch, type: "UInt64" + field :session_id, Ch, type: "UInt64" field :timestamp, :naive_datetime field :referrer, :string @@ -23,16 +23,16 @@ defmodule Plausible.ClickhouseEventV2 do field :utm_content, :string field :utm_term, :string - field :country_code, Ch.Types.FixedString, size: 2 - field :subdivision1_code, :string - field :subdivision2_code, :string - field :city_geoname_id, Ch.Types.UInt32 + field :country_code, Ch, type: "FixedString(2)" + field :subdivision1_code, Ch, type: "LowCardinality(String)" + field :subdivision2_code, Ch, type: "LowCardinality(String)" + field :city_geoname_id, Ch, type: "UInt32" - field :screen_size, :string - field :operating_system, :string - field :operating_system_version, :string - field :browser, :string - field :browser_version, :string + field :screen_size, Ch, type: "LowCardinality(String)" + field :operating_system, Ch, type: "LowCardinality(String)" + field :operating_system_version, Ch, type: "LowCardinality(String)" + field :browser, Ch, type: "LowCardinality(String)" + field :browser_version, Ch, type: "LowCardinality(String)" field :"meta.key", {:array, :string} field :"meta.value", {:array, :string} diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex index 9646527df..fd2628d48 100644 --- a/lib/plausible/clickhouse_session_v2.ex +++ b/lib/plausible/clickhouse_session_v2.ex @@ -4,21 +4,48 @@ defmodule Plausible.ClickhouseSessionV2 do """ use Ecto.Schema + defmodule BoolUInt8 do + @moduledoc """ + Custom type to cast Bool as UInt8 + """ + + use Ecto.Type + + u8 = Ecto.ParameterizedType.init(Ch, type: "UInt8") + + @impl true + def type, do: unquote(Macro.escape(u8)) + + @impl true + def cast(true), do: {:ok, 1} + def cast(false), do: {:ok, 0} + def cast(nil), do: {:ok, 0} + + @impl true + def load(1), do: {:ok, true} + def load(0), do: {:ok, false} + + @impl true + def dump(true), do: {:ok, 1} + def dump(false), do: {:ok, 0} + def dump(nil), do: {:ok, 0} + end + @primary_key false schema "sessions_v2" do field :hostname, :string - field :site_id, Ch.Types.UInt64 - field :user_id, Ch.Types.UInt64 - field :session_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" + field :user_id, Ch, type: "UInt64" + field :session_id, Ch, type: "UInt64" field :start, :naive_datetime - field :duration, Ch.Types.UInt32 - field :is_bounce, :boolean + field :duration, Ch, type: "UInt32" + field :is_bounce, BoolUInt8 field :entry_page, :string field :exit_page, :string - field :pageviews, Ch.Types.Int32 - field :events, Ch.Types.Int32 - field :sign, Ch.Types.Int8 + field :pageviews, Ch, type: "Int32" + field :events, Ch, type: "Int32" + field :sign, Ch, type: "Int8" field :"entry_meta.key", {:array, :string} field :"entry_meta.value", {:array, :string} @@ -31,16 +58,16 @@ defmodule Plausible.ClickhouseSessionV2 do field :referrer, :string field :referrer_source, :string - field :country_code, Ch.Types.FixedString, size: 2 - field :subdivision1_code, :string - field :subdivision2_code, :string - field :city_geoname_id, Ch.Types.UInt32 + field :country_code, Ch, type: "LowCardinality(FixedString(2))" + field :subdivision1_code, Ch, type: "LowCardinality(String)" + field :subdivision2_code, Ch, type: "LowCardinality(String)" + field :city_geoname_id, Ch, type: "UInt32" - field :screen_size, :string - field :operating_system, :string - field :operating_system_version, :string - field :browser, :string - field :browser_version, :string + field :screen_size, Ch, type: "LowCardinality(String)" + field :operating_system, Ch, type: "LowCardinality(String)" + field :operating_system_version, Ch, type: "LowCardinality(String)" + field :browser, Ch, type: "LowCardinality(String)" + field :browser_version, Ch, type: "LowCardinality(String)" field :timestamp, :naive_datetime field :transferred_from, :string diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex index 34cf0fdf2..d6aaf5f20 100644 --- a/lib/plausible/data_migration/numeric_ids.ex +++ b/lib/plausible/data_migration/numeric_ids.ex @@ -13,7 +13,7 @@ defmodule Plausible.DataMigration.NumericIDs do @primary_key false schema "domains_lookup" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :domain, :string end end diff --git a/lib/plausible/google/schemas.ex b/lib/plausible/google/schemas.ex index 780b692cc..80d52edf0 100644 --- a/lib/plausible/google/schemas.ex +++ b/lib/plausible/google/schemas.ex @@ -4,13 +4,13 @@ defmodule Plausible.Google.ImportedVisitor do @primary_key false schema "imported_visitors" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date - field :visitors, Ch.Types.UInt64 - field :pageviews, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 + field :visitors, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" end end @@ -20,17 +20,17 @@ defmodule Plausible.Google.ImportedSource do @primary_key false schema "imported_sources" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :source, :string field :utm_medium, :string field :utm_campaign, :string field :utm_content, :string field :utm_term, :string - field :visitors, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end @@ -40,14 +40,14 @@ defmodule Plausible.Google.ImportedPage do @primary_key false schema "imported_pages" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :hostname, :string field :page, :string - field :visitors, Ch.Types.UInt64 - field :pageviews, Ch.Types.UInt64 - field :exits, Ch.Types.UInt64 - field :time_on_page, Ch.Types.UInt64 + field :visitors, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :exits, Ch, type: "UInt64" + field :time_on_page, Ch, type: "UInt64" end end @@ -57,13 +57,13 @@ defmodule Plausible.Google.ImportedEntryPage do @primary_key false schema "imported_entry_pages" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :entry_page, :string - field :visitors, Ch.Types.UInt64 - field :entrances, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :visitors, Ch, type: "UInt64" + field :entrances, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end @@ -73,11 +73,11 @@ defmodule Plausible.Google.ImportedExitPage do @primary_key false schema "imported_exit_pages" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :exit_page, :string - field :visitors, Ch.Types.UInt64 - field :exits, Ch.Types.UInt64 + field :visitors, Ch, type: "UInt64" + field :exits, Ch, type: "UInt64" end end @@ -87,15 +87,15 @@ defmodule Plausible.Google.ImportedLocation do @primary_key false schema "imported_locations" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :country, :string field :region, :string - field :city, Ch.Types.UInt64 - field :visitors, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :city, Ch, type: "UInt64" + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end @@ -105,13 +105,13 @@ defmodule Plausible.Google.ImportedDevice do @primary_key false schema "imported_devices" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :device, :string - field :visitors, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end @@ -121,13 +121,13 @@ defmodule Plausible.Google.ImportedBrowser do @primary_key false schema "imported_browsers" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :browser, :string - field :visitors, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end @@ -137,12 +137,12 @@ defmodule Plausible.Google.ImportedOperatingSystem do @primary_key false schema "imported_operating_systems" do - field :site_id, Ch.Types.UInt64 + field :site_id, Ch, type: "UInt64" field :date, :date field :operating_system, :string - field :visitors, Ch.Types.UInt64 - field :visits, Ch.Types.UInt64 - field :visit_duration, Ch.Types.UInt64 - field :bounces, Ch.Types.UInt32 + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" end end diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.ex index 941be5c9a..8b79d721c 100644 --- a/lib/plausible/ingestion/counters/record.ex +++ b/lib/plausible/ingestion/counters/record.ex @@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do @primary_key false schema "ingest_counters" do field :event_timebucket, :utc_datetime - field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64 - field :domain, :string - field :metric, :string - field :value, Ch.Types.UInt64 + field :site_id, Ch, type: "Nullable(UInt64)" + field :domain, Ch, type: "LowCardinality(String)" + field :metric, Ch, type: "LowCardinality(String)" + field :value, Ch, type: "UInt64" end end diff --git a/mix.exs b/mix.exs index 4eae9d830..46b60bd5b 100644 --- a/mix.exs +++ b/mix.exs @@ -61,7 +61,7 @@ defmodule Plausible.MixProject do {:bcrypt_elixir, "~> 3.0"}, {:bypass, "~> 2.1", only: [:dev, :test]}, {:cachex, "~> 3.4"}, - {:chto, github: "plausible/chto"}, + {:ecto_ch, "~> 0.1.0"}, {:combination, "~> 0.0.3"}, {:connection, "~> 1.1", override: true}, {:cors_plug, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 206cc4e9b..6bb9d7b26 100644 --- a/mix.lock +++ b/mix.lock @@ -10,9 +10,8 @@ "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "ch": {:git, "https://github.com/plausible/ch.git", "f411daa07c6310d3308f81397905df65330aeb64", []}, + "ch": {:hex, :ch, "0.1.14", "c53489b66eeb83dca63e63155c3e0de74f99ba30a15e90d0cd6b38db86be5891", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "306357bd9a92662713b6b9b4244eb94e1ef93ececa3906dbbef7392c4001c8ef"}, "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, - "chto": {:git, "https://github.com/plausible/chto.git", "6afe76281e617c19df9b7a705c8401f604fe091f", []}, "cldr_utils": {:hex, :cldr_utils, "2.22.0", "5df60df2603dfeeffe26e40ab1ee565df34da288a53bb2db0d0dbd243fd646ef", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "ea14e8a6aa89ffd59a5d49baebe7ebf852cc024ac50dc2b3dabcd3786eeed657"}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, @@ -26,12 +25,13 @@ "csv": {:hex, :csv, "2.5.0", "c47b5a5221bf2e56d6e8eb79e77884046d7fd516280dc7d9b674251e0ae46246", [:mix], [{:parallel_stream, "~> 1.0.4 or ~> 1.1.0", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "e821f541487045c7591a1963eeb42afff0dfa99bdcdbeb3410795a2f59c77d34"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, - "decimal": {:hex, :decimal, "2.1.0", "4a607736e18d9a77ea44fa18388bf43bb7f7df70ff2e0d23d4238f9d18a3ad7d", [:mix], [], "hexpm", "6396853b6b4bf8b1ff2c2217121f1fad4a1aff999014a10fba60e6009194f91b"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, "digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"}, "double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"}, "earmark_parser": {:hex, :earmark_parser, "1.4.31", "a93921cdc6b9b869f519213d5bc79d9e218ba768d7270d46fdcf1c01bacff9e2", [:mix], [], "hexpm", "317d367ee0335ef037a87e46c91a2269fef6306413f731e8ec11fc45a7efd059"}, "ecto": {:hex, :ecto, "3.9.5", "9f0aa7ae44a1577b651c98791c6988cd1b69b21bc724e3fd67090b97f7604263", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d4f3115d8cbacdc0bfa4b742865459fb1371d0715515842a1fb17fe31920b74c"}, + "ecto_ch": {:hex, :ecto_ch, "0.1.9", "cb8e7bbe926c73d4160f4a1d9153a7bca5b757678b648ff2d8baf7c0dad11200", [:mix], [{:ch, "~> 0.1.14", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "94c125d335581208c64b76a37d98961393f81131f8ef2a57ea4c4e66bbe42753"}, "ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"}, "elixir_make": {:hex, :elixir_make, "0.7.6", "67716309dc5d43e16b5abbd00c01b8df6a0c2ab54a8f595468035a50189f9169", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5a0569756b0f7873a77687800c164cca6dfc03a09418e6fcf853d78991f49940"}, "envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"},