From 6637751a5e83f890e0e126b04708c661a6785d56 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 23 Mar 2023 09:47:41 +0100 Subject: [PATCH] Implement Numeric IDs migration (#2762) * Implement Numeric IDs migration * Fix typo * Mute credo for now * Improve configurability and add stop_t * Adjust to Ch/Chto only * Fix opts key for dictionary password * Add regular ecto migration with numeric ids v2 schemas (#2768) * Add regular ecto migration * Fix typo * Update priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs Co-authored-by: Vini Brasil * Implement v2 events/sessions schema modules (#2777) * Implement v2 events/sessions schema modules * Clean up session schemas --------- Co-authored-by: Vini Brasil * Update moduledocs --------- Co-authored-by: Vini Brasil --- ...ickhouse_schema.ex => clickhouse_event.ex} | 0 lib/plausible/clickhouse_event_v2.ex | 76 ++++++++++ ...khouse_schema.ex => clickhouse_session.ex} | 32 ---- lib/plausible/clickhouse_session_v2.ex | 52 +++++++ lib/plausible/data_migration.ex | 64 ++++++++ lib/plausible/data_migration/numeric_ids.ex | 141 ++++++++++++++++++ lib/plausible/data_migration/repo.ex | 22 +++ .../sql/attach-tmp-events-v2.sql.eex | 1 + .../sql/attach-tmp-sessions-v2.sql.eex | 1 + .../NumericIDs/sql/check-replicas.sql.eex | 1 + .../sql/create-dict-from-static-file.sql.eex | 9 ++ .../NumericIDs/sql/create-events-v2.sql.eex | 39 +++++ .../NumericIDs/sql/create-sessions-v2.sql.eex | 45 ++++++ .../sql/create-tmp-events-v2.sql.eex | 6 + .../sql/create-tmp-sessions-v2.sql.eex | 6 + .../NumericIDs/sql/drop-dict.sql.eex | 1 + .../NumericIDs/sql/drop-events-v2.sql.eex | 1 + .../NumericIDs/sql/drop-sessions-v2.sql.eex | 1 + .../NumericIDs/sql/drop-tmp-events-v2.sql.eex | 1 + .../sql/drop-tmp-sessions-v2.sql.eex | 1 + .../sql/insert-into-tmp-events-v2.sql.eex | 54 +++++++ .../sql/insert-into-tmp-sessions-v2.sql.eex | 30 ++++ .../NumericIDs/sql/list-partitions.sql.eex | 6 + .../sql/truncate-tmp-events-v2.sql.eex | 1 + .../sql/truncate-tmp-sessions-v2.sql.eex | 1 + .../20230320094327_create_v2_schemas.exs | 33 ++++ 26 files changed, 593 insertions(+), 32 deletions(-) rename lib/plausible/{event/clickhouse_schema.ex => clickhouse_event.ex} (100%) create mode 100644 lib/plausible/clickhouse_event_v2.ex rename lib/plausible/{session/clickhouse_schema.ex => clickhouse_session.ex} (66%) create mode 100644 lib/plausible/clickhouse_session_v2.ex create mode 100644 lib/plausible/data_migration.ex create mode 100644 lib/plausible/data_migration/numeric_ids.ex create mode 100644 lib/plausible/data_migration/repo.ex create mode 100644 priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex create mode 100644 priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex create mode 100644 priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/clickhouse_event.ex similarity index 100% rename from lib/plausible/event/clickhouse_schema.ex rename to lib/plausible/clickhouse_event.ex diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex new file mode 100644 index 000000000..a404d5e43 --- /dev/null +++ b/lib/plausible/clickhouse_event_v2.ex @@ -0,0 +1,76 @@ +defmodule Plausible.ClickhouseEventV2 do + @moduledoc """ + Event schema for when NumericIDs migration is complete + """ + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + schema "events_v2" do + field :name, :string + field :site_id, Ch.Types.UInt64 + field :hostname, :string + field :pathname, :string + field :user_id, Ch.Types.UInt64 + field :session_id, Ch.Types.UInt64 + field :timestamp, :naive_datetime + + field :referrer, :string + field :referrer_source, :string + field :utm_medium, :string + field :utm_source, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string + + field :country_code, Ch.Types.FixedString, size: 2 + field :subdivision1_code, :string + field :subdivision2_code, :string + field :city_geoname_id, Ch.Types.UInt32 + + field :screen_size, :string + field :operating_system, :string + field :operating_system_version, :string + field :browser, :string + field :browser_version, :string + + field :"meta.key", {:array, :string} + field :"meta.value", {:array, :string} + field :transferred_from, :string + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :name, + :site_id, + :hostname, + :pathname, + :user_id, + :timestamp, + :operating_system, + :operating_system_version, + :browser, + :browser_version, + :referrer, + :referrer_source, + :utm_medium, + :utm_source, + :utm_campaign, + :utm_content, + :utm_term, + :country_code, + :subdivision1_code, + :subdivision2_code, + :city_geoname_id, + :screen_size, + :"meta.key", + :"meta.value" + ], + empty_values: [nil, ""] + ) + |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp]) + end +end diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/clickhouse_session.ex similarity index 66% rename from lib/plausible/session/clickhouse_schema.ex rename to lib/plausible/clickhouse_session.ex index 256fac3fe..5f5a3c02b 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/clickhouse_session.ex @@ -1,6 +1,5 @@ defmodule Plausible.ClickhouseSession do use Ecto.Schema - import Ecto.Changeset @primary_key false schema "sessions" do @@ -47,35 +46,4 @@ defmodule Plausible.ClickhouseSession do def random_uint64() do :crypto.strong_rand_bytes(8) |> :binary.decode_unsigned() end - - def changeset(session, attrs) do - session - |> cast(attrs, [ - :hostname, - :domain, - :entry_page, - :exit_page, - :fingerprint, - :start, - :length, - :is_bounce, - :operating_system, - :operating_system_version, - :browser_version, - :referrer, - :referrer_source, - :utm_medium, - :utm_source, - :utm_campaign, - :utm_content, - :utm_term, - :country_code, - :country_geoname_id, - :subdivision1_code, - :subdivision2_code, - :city_geoname_id, - :screen_size - ]) - |> validate_required([:hostname, :domain, :fingerprint, :is_bounce, :start]) - end end diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex new file mode 100644 index 000000000..9646527df --- /dev/null +++ b/lib/plausible/clickhouse_session_v2.ex @@ -0,0 +1,52 @@ +defmodule Plausible.ClickhouseSessionV2 do + @moduledoc """ + Session schema for when NumericIDs migration is complete + """ + use Ecto.Schema + + @primary_key false + schema "sessions_v2" do + field :hostname, :string + field :site_id, Ch.Types.UInt64 + field :user_id, Ch.Types.UInt64 + field :session_id, Ch.Types.UInt64 + + field :start, :naive_datetime + field :duration, Ch.Types.UInt32 + field :is_bounce, :boolean + field :entry_page, :string + field :exit_page, :string + field :pageviews, Ch.Types.Int32 + field :events, Ch.Types.Int32 + field :sign, Ch.Types.Int8 + + field :"entry_meta.key", {:array, :string} + field :"entry_meta.value", {:array, :string} + + field :utm_medium, :string + field :utm_source, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string + field :referrer, :string + field :referrer_source, :string + + field :country_code, Ch.Types.FixedString, size: 2 + field :subdivision1_code, :string + field :subdivision2_code, :string + field :city_geoname_id, Ch.Types.UInt32 + + field :screen_size, :string + field :operating_system, :string + field :operating_system_version, :string + field :browser, :string + field :browser_version, :string + field :timestamp, :naive_datetime + + field :transferred_from, :string + end + + def random_uint64() do + :crypto.strong_rand_bytes(8) |> :binary.decode_unsigned() + end +end diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex new file mode 100644 index 000000000..611826cef --- /dev/null +++ b/lib/plausible/data_migration.ex @@ -0,0 +1,64 @@ +defmodule Plausible.DataMigration do + @moduledoc """ + Base module for coordinated Clickhouse data migrations + run via remote shell or otherwise (TBD). + """ + + defmacro __using__(opts) do + dir = Keyword.fetch!(opts, :dir) + repo = Keyword.get(opts, :repo, Plausible.DataMigration.Repo) + + quote bind_quoted: [dir: dir, repo: repo] do + @dir dir + @repo repo + + def run_sql_confirm(name, assigns \\ []) do + query = unwrap(name, assigns) + + confirm("Execute?", fn -> do_run(name, query) end) + end + + def confirm(message, func) do + prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset() + + if String.downcase(String.trim(IO.gets(prompt))) == "n" do + IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}") + {:ok, :skip} + else + func.() + end + end + + defp unwrap(name, assigns) do + "priv/data_migrations" + |> Path.join(@dir) + |> Path.join("sql") + |> Path.join(name <> ".sql.eex") + |> EEx.eval_file(assigns: assigns) + end + + def run_sql(name, assigns \\ []) do + query = unwrap(name, assigns) + do_run(name, query) + end + + defp do_run(name, query) do + {:ok, res} = @repo.query(query, [], timeout: :infinity) + IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n") + IO.puts(String.duplicate("-", 78)) + {:ok, res} + end + + defp unwrap_with_io(name, assigns) do + IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}") + query = unwrap(name, assigns) + + IO.puts(""" + -> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()} + """) + + query + end + end + end +end diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex new file mode 100644 index 000000000..8782df7a2 --- /dev/null +++ b/lib/plausible/data_migration/numeric_ids.ex @@ -0,0 +1,141 @@ +defmodule Plausible.DataMigration.NumericIDs do + @moduledoc """ + Numeric IDs migration, SQL files available at: + priv/data_migrations/NumericIDs/sql + """ + use Plausible.DataMigration, dir: "NumericIDs" + + @table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'" + + # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + def run(opts \\ []) do + interactive? = Keyword.get(opts, :interactive?, true) + + db_url = + System.get_env( + "NUMERIC_IDS_MIGRATION_DB_URL", + Application.get_env(:plausible, Plausible.IngestRepo)[:url] + ) + + max_threads = + "NUMERIC_IDS_MIGRATION_MAX_THREADS" |> System.get_env("16") |> String.to_integer() + + # TBD: There's most likely a bug in Clickhouse defining Postgres dictionaries, + # we'll use a static URL for now + dict_url = Keyword.get(opts, :dict_url) || System.get_env("DOMAINS_DICT_URL") || "" + + dict_password = + Keyword.get(opts, :dict_password) || System.get_env("DOMAINS_DICT_PASSWORD") || "" + + table_settings = + Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") || + @table_settings + + start_from = + Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM") + + stop_at = + Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT") || + previous_part() + + (byte_size(dict_url) > 0 and byte_size(dict_password) > 0) || + raise "Set DOMAINS_DICT_URL and DOMAINS_DICT_PASSWORD" + + @repo.start(db_url, max_threads) + + cluster? = + case run_sql("check-replicas") do + {:ok, %{num_rows: 0}} -> false + {:ok, %{num_rows: 1}} -> true + end + + {:ok, %{rows: partitions}} = + run_sql("list-partitions", start_from: start_from, stop_at: stop_at) + + partitions = Enum.map(partitions, fn [part] -> part end) + + start_from = start_from || List.first(partitions) + + IO.puts(""" + Got the following migration settings: + + - max_threads: #{max_threads} + - dict_url: #{dict_url} + - dict_password: ✅ + - table_settings: #{table_settings} + - db url: #{db_url} + - cluster?: #{cluster?} + - partitions to do: #{inspect(partitions, pretty: true, width: 80)} + - start from: #{start_from} + - stop at: #{stop_at} + """) + + run_sql_fn = + if interactive? do + &run_sql_confirm/2 + else + &run_sql/2 + end + + confirm_fn = + if interactive? do + &confirm/2 + else + fn _, run_fn -> + run_fn.() + end + end + + {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?) + {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?) + {:ok, _} = run_sql_fn.("drop-tmp-events-v2", []) + {:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", []) + {:ok, _} = run_sql_fn.("drop-dict", []) + + {:ok, _} = + run_sql_fn.("create-dict-from-static-file", + dict_url: dict_url, + dict_password: dict_password + ) + + {:ok, _} = run_sql_fn.("create-events-v2", table_settings: table_settings, cluster?: cluster?) + + {:ok, _} = + run_sql_fn.("create-sessions-v2", table_settings: table_settings, cluster?: cluster?) + + {:ok, _} = run_sql_fn.("create-tmp-events-v2", table_settings: table_settings) + {:ok, _} = run_sql_fn.("create-tmp-sessions-v2", table_settings: table_settings) + + confirm_fn.("Start migration? (starting from partition: #{start_from})", fn -> + IO.puts("start.. #{DateTime.utc_now()}") + + for part <- partitions do + part_start = System.monotonic_time() + + confirm_fn.("Run partition: #{part}?", fn -> + {:ok, _} = run_sql("insert-into-tmp-events-v2", partition: part) + {:ok, _} = run_sql("attach-tmp-events-v2", partition: part) + {:ok, _} = run_sql("truncate-tmp-events-v2", []) + {:ok, _} = run_sql("insert-into-tmp-sessions-v2", partition: part) + {:ok, _} = run_sql("attach-tmp-sessions-v2", partition: part) + {:ok, _} = run_sql("truncate-tmp-sessions-v2", []) + end) + + part_end = System.monotonic_time() + + IO.puts( + "#{part} took #{System.convert_time_unit(part_end - part_start, :native, :second)} seconds" + ) + end + + IO.puts("end.. #{DateTime.utc_now()}") + end) + end + + defp previous_part() do + now = NaiveDateTime.utc_now() + month = String.pad_leading("#{now.month - 1}", 2, "0") + year = "#{now.year}" + "#{year}#{month}" + end +end diff --git a/lib/plausible/data_migration/repo.ex b/lib/plausible/data_migration/repo.ex new file mode 100644 index 000000000..3ceac17b2 --- /dev/null +++ b/lib/plausible/data_migration/repo.ex @@ -0,0 +1,22 @@ +defmodule Plausible.DataMigration.Repo do + @moduledoc """ + Ecto.Repo for Clickhouse data migrations, to be started manually, + outside of the main application supervision tree. + """ + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.ClickHouse + + def start(url, max_threads) when is_binary(url) and is_integer(max_threads) do + start_link( + url: url, + queue_target: 500, + queue_interval: 2000, + pool_size: 1, + settings: [ + max_insert_threads: max_threads, + send_progress_in_http_headers: 1 + ] + ) + end +end diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex new file mode 100644 index 000000000..ca274978f --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +ALTER TABLE events_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_events_v2 diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex new file mode 100644 index 000000000..1b64580f9 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +ALTER TABLE sessions_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_sessions_v2 diff --git a/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex new file mode 100644 index 000000000..6d639edcf --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex @@ -0,0 +1 @@ +SELECT 1 FROM system.replicas WHERE table = 'events' diff --git a/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex new file mode 100644 index 000000000..be5c45a21 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex @@ -0,0 +1,9 @@ +CREATE DICTIONARY domains_lookup ( +`id` UInt64, +`site_id` UInt64, +`domain` String, +`partition` String +) PRIMARY KEY domain +SOURCE(HTTP(URL '<%= @dict_url %>' CREDENTIALS(user 'user' password '<%= @dict_password %>') FORMAT 'CSVWithNames')) +LIFETIME(3600) +LAYOUT(COMPLEX_KEY_HASHED()) diff --git a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex new file mode 100644 index 000000000..6413354cd --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex @@ -0,0 +1,39 @@ +CREATE TABLE events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `timestamp` DateTime CODEC(Delta(4), LZ4), + `name` LowCardinality(String), + `site_id` UInt64, + `user_id` UInt64, + `session_id` UInt64, + `hostname` String, + `pathname` String CODEC(ZSTD(3)), + `referrer` String, + `referrer_source` String, + `country_code` FixedString(2), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `meta.key` Array(String), + `meta.value` Array(String), + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String +) +<%= if @cluster? do %> +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/events_v2', '{replica}') +<% else %> +ENGINE = MergeTree() +<% end %> +PARTITION BY toYYYYMM(timestamp) +PRIMARY KEY (site_id, toDate(timestamp), name, user_id) +ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex new file mode 100644 index 000000000..38d1e5eeb --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex @@ -0,0 +1,45 @@ +CREATE TABLE sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `session_id` UInt64, + `sign` Int8, + `site_id` UInt64, + `user_id` UInt64, + `hostname` String, + `timestamp` DateTime CODEC(DoubleDelta, LZ4), + `start` DateTime CODEC(DoubleDelta, LZ4), + `is_bounce` UInt8, + `entry_page` String, + `exit_page` String, + `pageviews` Int32, + `events` Int32, + `duration` UInt32, + `referrer` String, + `referrer_source` String, + `country_code` LowCardinality(FixedString(2)), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String, + `entry_meta.key` Array(String), + `entry_meta.value` Array(String) +) +<%= if @cluster? do %> +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2', '{replica}', sign) +<% else %> +ENGINE = CollapsingMergeTree(sign) +<% end %> +PARTITION BY toYYYYMM(start) +PRIMARY KEY (site_id, toDate(start), user_id, session_id) +ORDER BY (site_id, toDate(start), user_id, session_id) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex new file mode 100644 index 000000000..be70d6c44 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS + tmp_events_v2 AS events_v2 ENGINE = MergeTree() + ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp) + PRIMARY KEY (site_id, toDate(timestamp), name, user_id) + PARTITION BY toYYYYMM(timestamp) + <%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex new file mode 100644 index 000000000..bc93a129b --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS + tmp_sessions_v2 AS sessions_v2 ENGINE = CollapsingMergeTree(sign) + PARTITION BY toYYYYMM(start) + PRIMARY KEY (site_id, toDate(start), user_id, session_id) + ORDER BY (site_id, toDate(start), user_id, session_id) + <%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex new file mode 100644 index 000000000..3e4f22643 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex @@ -0,0 +1 @@ +DROP DICTIONARY IF EXISTS domains_lookup SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex new file mode 100644 index 000000000..ae3927e49 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex new file mode 100644 index 000000000..eee766842 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex new file mode 100644 index 000000000..9ce47fbae --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS tmp_events_v2 SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex new file mode 100644 index 000000000..ea3666c51 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS tmp_sessions_v2 SYNC diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex new file mode 100644 index 000000000..454ac0996 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex @@ -0,0 +1,54 @@ +INSERT INTO tmp_events_v2 ( + site_id, timestamp, name, user_id, + session_id, hostname, pathname, referrer, + referrer_source, country_code, screen_size, + operating_system, browser, utm_medium, + utm_source, utm_campaign, meta.key, + meta.value, browser_version, operating_system_version, + subdivision1_code, subdivision2_code, + city_geoname_id, utm_content, utm_term, + transferred_from +) +SELECT + dictGet( + 'domains_lookup', 'site_id', domain + ), + timestamp, + name, + user_id, + session_id, + hostname, + pathname, + referrer, + referrer_source, + country_code, + screen_size, + operating_system, + browser, + utm_medium, + utm_source, + utm_campaign, + meta.key, + meta.value, + browser_version, + operating_system_version, + subdivision1_code, + subdivision2_code, + city_geoname_id, + utm_content, + utm_term, + transferred_from +FROM + events +WHERE + ( + domain IN ( + SELECT + domain + FROM + dictionary('domains_lookup') + WHERE + partition <= '<%= @partition %>' + ) + ) +AND (_partition_id = '<%= @partition %>') diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex new file mode 100644 index 000000000..194e877af --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex @@ -0,0 +1,30 @@ +INSERT INTO tmp_sessions_v2 ( +site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page, +exit_page, pageviews, events, duration, referrer, referrer_source, country_code, +screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version, +operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content, +utm_term, transferred_from, entry_meta.key, entry_meta.value +) +SELECT + dictGet( + 'domains_lookup', 'site_id', domain + ), + session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page, + exit_page, pageviews, events, duration, referrer, referrer_source, country_code, + screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version, + operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content, + utm_term, transferred_from, entry_meta.key, entry_meta.value +FROM + sessions +WHERE + ( + domain IN ( + SELECT + domain + FROM + dictionary('domains_lookup') + WHERE + partition <= '<%= @partition %>' + ) + ) +AND (_partition_id = '<%= @partition %>') diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex new file mode 100644 index 000000000..c945e7376 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex @@ -0,0 +1,6 @@ +SELECT partition +FROM system.parts +WHERE table = 'events' +<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %> +<%= if @stop_at do %>AND partition <= '<%= @stop_at %>'<% end %> +ORDER BY partition diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex new file mode 100644 index 000000000..18e6fef98 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +TRUNCATE TABLE tmp_events_v2 diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex new file mode 100644 index 000000000..22b8bbddf --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +TRUNCATE TABLE tmp_sessions_v2 diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs new file mode 100644 index 000000000..a77591f64 --- /dev/null +++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs @@ -0,0 +1,33 @@ +defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do + @moduledoc """ + Normally, for live environments the migration will be done via + `DataMigration.NumericIDs` module (TBD). In which case PASS_V2_SCHEMA_MIGRATION + environment variable needs to be set, to only make the standard migrate + command write an entry into schema_migrations. + + For tests, and entirely new small, self-hosted instances however, + we want to keep the ability of preparing the database without enforcing + any data migration. + """ + + use Ecto.Migration + + use Plausible.DataMigration, dir: "NumericIDs" + + @cluster? false + @settings "SETTINGS index_granularity = 8192" + + def up do + if System.get_env("PASS_V2_SCHEMA_MIGRATION") do + :ok + else + execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?) + execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?) + end + end + + def down do + execute unwrap("drop-events-v2", cluster?: @cluster?) + execute unwrap("drop-sessions-v2", cluster?: @cluster?) + end +end