Add data migration for moving sessions_v2 table to VersionedCollapsingMergeTree (#3802)
* Add data migration for moving to VersionedCollapsingMergeTree This has been tested locally and partially on staging. Still requires a bit of work to verify. Verification query: ``` SELECT main._partition_id, tmp.count, main.count FROM ( SELECT _partition_id, count() AS count FROM sessions_v2_tmp_versioned GROUP BY _partition_id ) AS tmp FULL OUTER JOIN ( SELECT _partition_id, count() AS count FROM sessions_v2 GROUP BY _partition_id ) AS main ON (tmp._partition_id == main._partition_id) ORDER BY main._partition_id ``` * Add an early exit to migration * cluster? extract common codepull/3806/head^2 r2024.02.22-365
parent
d74b1d5e60
commit
8ba2b934b3
@ -0,0 +1,75 @@
|
||||
defmodule Plausible.DataMigration.VersionedSessions do
|
||||
@moduledoc """
|
||||
Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
|
||||
priv/data_migrations/VersionedSessions/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
|
||||
|
||||
@suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
|
||||
@versioned_table_engines [
|
||||
"ReplicatedVersionedCollapsingMergeTree",
|
||||
"VersionedCollapsingMergeTree"
|
||||
]
|
||||
|
||||
def run(opts \\ []) do
|
||||
run_exchange? = Keyword.get(opts, :run_exchange?, true)
|
||||
|
||||
unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
|
||||
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
|
||||
|
||||
{:ok, %{rows: partitions}} = run_sql("list-partitions")
|
||||
partitions = Enum.map(partitions, fn [part] -> part end)
|
||||
|
||||
{:ok, %{rows: [[current_table_engine, table_settings]]}} =
|
||||
run_sql("get-sessions-table-settings")
|
||||
|
||||
if Enum.member?(@versioned_table_engines, current_table_engine) do
|
||||
IO.puts("sessions_v2 table is already versioned, no migration needed")
|
||||
else
|
||||
{:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("create-sessions-tmp-table",
|
||||
cluster?: cluster?,
|
||||
table_settings: table_settings,
|
||||
unique_suffix: unique_suffix
|
||||
)
|
||||
|
||||
for partition <- partitions do
|
||||
{:ok, _} = run_sql("attach-partition", partition: partition)
|
||||
end
|
||||
|
||||
if run_exchange? do
|
||||
run_exchange(cluster?)
|
||||
end
|
||||
|
||||
IO.puts("Migration done!")
|
||||
end
|
||||
end
|
||||
|
||||
defp run_exchange(cluster?) do
|
||||
case run_sql("exchange-sessions-tables", cluster?: cluster?) do
|
||||
{:ok, _} ->
|
||||
nil
|
||||
|
||||
# Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
|
||||
{:error, %Ch.Error{code: 1}} ->
|
||||
IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2",
|
||||
to: "sessions_v2_backup",
|
||||
cluster?: cluster?
|
||||
)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2_tmp_versioned",
|
||||
to: "sessions_v2",
|
||||
cluster?: cluster?
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1 @@
|
||||
ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2
|
||||
@ -0,0 +1 @@
|
||||
SELECT 1 FROM system.replicas WHERE table = 'sessions_v2'
|
||||
@ -0,0 +1,13 @@
|
||||
CREATE TABLE sessions_v2_tmp_versioned
|
||||
<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
AS sessions_v2
|
||||
<%= if @cluster? do %>
|
||||
ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events)
|
||||
<% else %>
|
||||
ENGINE = VersionedCollapsingMergeTree(sign, events)
|
||||
<% end %>
|
||||
PARTITION BY toYYYYMM(start)
|
||||
PRIMARY KEY (site_id, toDate(start), user_id, session_id)
|
||||
ORDER BY (site_id, toDate(start), user_id, session_id)
|
||||
SAMPLE BY user_id
|
||||
<%= @table_settings %>
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
@ -0,0 +1 @@
|
||||
EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
@ -0,0 +1,4 @@
|
||||
SELECT engine, extract(engine_full, 'SETTINGS .+')
|
||||
FROM system.tables
|
||||
WHERE database = currentDatabase()
|
||||
AND name = 'sessions_v2'
|
||||
@ -0,0 +1,5 @@
|
||||
SELECT distinct(partition)
|
||||
FROM system.parts
|
||||
WHERE database = currentDatabase()
|
||||
AND table = 'sessions_v2'
|
||||
ORDER BY partition
|
||||
@ -0,0 +1 @@
|
||||
RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
Loading…
Reference in New Issue