From 8ba2b934b36993c8a9e9281ffde770821166ac15 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 22 Feb 2024 09:54:39 +0200 Subject: [PATCH] Add data migration for moving sessions_v2 table to VersionedCollapsingMergeTree (#3802) * Add data migration for moving to VersionedCollapsingMergeTree This has been tested locally and partially on staging. Still requires a bit of work to verify. Verification query: ``` SELECT main._partition_id, tmp.count, main.count FROM ( SELECT _partition_id, count() AS count FROM sessions_v2_tmp_versioned GROUP BY _partition_id ) AS tmp FULL OUTER JOIN ( SELECT _partition_id, count() AS count FROM sessions_v2 GROUP BY _partition_id ) AS main ON (tmp._partition_id == main._partition_id) ORDER BY main._partition_id ``` * Add an early exit to migration * cluster? extract common code --- lib/plausible/data_migration.ex | 13 +++- .../data_migration/versioned_sessions.ex | 75 +++++++++++++++++++ lib/plausible/migration_utils.ex | 8 +- .../sql/attach-partition.sql.eex | 1 + .../sql/check-replicas.sql.eex | 1 + .../sql/create-sessions-tmp-table.sql.eex | 13 ++++ .../sql/drop-sessions-tmp-table.sql.eex | 1 + .../sql/exchange-sessions-tables.sql.eex | 1 + .../sql/get-sessions-table-settings.sql.eex | 4 + .../sql/list-partitions.sql.eex | 5 ++ .../sql/rename-table.sql.eex | 1 + 11 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 lib/plausible/data_migration/versioned_sessions.ex create mode 100644 priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex create mode 100644 priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex index 7655f56e3..bc41dba9a 100644 --- a/lib/plausible/data_migration.ex +++ b/lib/plausible/data_migration.ex @@ -68,10 +68,15 @@ defmodule Plausible.DataMigration do end defp do_run(name, query) do - {:ok, res} = @repo.query(query, [], timeout: :infinity) - IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n") - IO.puts(String.duplicate("-", 78)) - {:ok, res} + case @repo.query(query, [], timeout: :infinity) do + {:ok, res} -> + IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n") + IO.puts(String.duplicate("-", 78)) + {:ok, res} + + result -> + result + end end defp unwrap_with_io(name, assigns) do diff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.ex new file mode 100644 index 000000000..1c6e18299 --- /dev/null +++ b/lib/plausible/data_migration/versioned_sessions.ex @@ -0,0 +1,75 @@ +defmodule Plausible.DataMigration.VersionedSessions do + @moduledoc """ + Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at: + priv/data_migrations/VersionedSessions/sql + """ + use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo + + @suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}" + @versioned_table_engines [ + "ReplicatedVersionedCollapsingMergeTree", + "VersionedCollapsingMergeTree" + ] + + def run(opts \\ []) do + run_exchange? = Keyword.get(opts, :run_exchange?, true) + + unique_suffix = Timex.now() |> Timex.format!(@suffix_format) + + cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2") + + {:ok, %{rows: partitions}} = run_sql("list-partitions") + partitions = Enum.map(partitions, fn [part] -> part end) + + {:ok, %{rows: [[current_table_engine, table_settings]]}} = + run_sql("get-sessions-table-settings") + + if Enum.member?(@versioned_table_engines, current_table_engine) do + IO.puts("sessions_v2 table is already versioned, no migration needed") + else + {:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?) + + {:ok, _} = + run_sql("create-sessions-tmp-table", + cluster?: cluster?, + table_settings: table_settings, + unique_suffix: unique_suffix + ) + + for partition <- partitions do + {:ok, _} = run_sql("attach-partition", partition: partition) + end + + if run_exchange? do + run_exchange(cluster?) + end + + IO.puts("Migration done!") + end + end + + defp run_exchange(cluster?) do + case run_sql("exchange-sessions-tables", cluster?: cluster?) do + {:ok, _} -> + nil + + # Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap + {:error, %Ch.Error{code: 1}} -> + IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically") + + {:ok, _} = + run_sql("rename-table", + from: "sessions_v2", + to: "sessions_v2_backup", + cluster?: cluster? + ) + + {:ok, _} = + run_sql("rename-table", + from: "sessions_v2_tmp_versioned", + to: "sessions_v2", + cluster?: cluster? + ) + end + end +end diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex index a4daa72e2..ac416a389 100644 --- a/lib/plausible/migration_utils.ex +++ b/lib/plausible/migration_utils.ex @@ -4,9 +4,13 @@ defmodule Plausible.MigrationUtils do """ def on_cluster_statement(table) do + if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "") + end + + def clustered_table?(table) do case Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do - {:ok, %{rows: []}} -> "" - {:ok, _} -> "ON CLUSTER '{cluster}'" + {:ok, %{rows: []}} -> false + {:ok, _} -> true end end end diff --git a/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex new file mode 100644 index 000000000..c1d8cfb5f --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex @@ -0,0 +1 @@ +ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2 diff --git a/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex new file mode 100644 index 000000000..367a574df --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex @@ -0,0 +1 @@ +SELECT 1 FROM system.replicas WHERE table = 'sessions_v2' diff --git a/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex new file mode 100644 index 000000000..dc579c858 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex @@ -0,0 +1,13 @@ +CREATE TABLE sessions_v2_tmp_versioned +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +AS sessions_v2 +<%= if @cluster? do %> +ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events) +<% else %> +ENGINE = VersionedCollapsingMergeTree(sign, events) +<% end %> +PARTITION BY toYYYYMM(start) +PRIMARY KEY (site_id, toDate(start), user_id, session_id) +ORDER BY (site_id, toDate(start), user_id, session_id) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex new file mode 100644 index 000000000..7a967981a --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex new file mode 100644 index 000000000..416da043e --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex @@ -0,0 +1 @@ +EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex new file mode 100644 index 000000000..ffe35fb43 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex @@ -0,0 +1,4 @@ +SELECT engine, extract(engine_full, 'SETTINGS .+') +FROM system.tables +WHERE database = currentDatabase() + AND name = 'sessions_v2' diff --git a/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex new file mode 100644 index 000000000..986822688 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex @@ -0,0 +1,5 @@ +SELECT distinct(partition) +FROM system.parts +WHERE database = currentDatabase() + AND table = 'sessions_v2' +ORDER BY partition diff --git a/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex new file mode 100644 index 000000000..ea07c8d93 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex @@ -0,0 +1 @@ +RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>