Add data migration for moving sessions_v2 table to VersionedCollapsingMergeTree (#3802)

* Add data migration for moving to VersionedCollapsingMergeTree

This has been tested locally and partially on staging. Still requires a bit of work to verify.

Verification query:

```
SELECT main._partition_id, tmp.count, main.count
FROM (
SELECT _partition_id, count() AS count
FROM sessions_v2_tmp_versioned
GROUP BY _partition_id
) AS tmp
FULL OUTER JOIN (
SELECT _partition_id, count() AS count
FROM sessions_v2
GROUP BY _partition_id
) AS main
ON (tmp._partition_id == main._partition_id)
ORDER BY main._partition_id
```

* Add an early exit to migration

* cluster? extract common code
pull/3806/head^2 r2024.02.22-365
Karl-Aksel Puulmann 3 years ago committed by GitHub
parent d74b1d5e60
commit 8ba2b934b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -68,10 +68,15 @@ defmodule Plausible.DataMigration do
end
defp do_run(name, query) do
{:ok, res} = @repo.query(query, [], timeout: :infinity)
IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
IO.puts(String.duplicate("-", 78))
{:ok, res}
case @repo.query(query, [], timeout: :infinity) do
{:ok, res} ->
IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
IO.puts(String.duplicate("-", 78))
{:ok, res}
result ->
result
end
end
defp unwrap_with_io(name, assigns) do

@ -0,0 +1,75 @@
defmodule Plausible.DataMigration.VersionedSessions do
@moduledoc """
Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
priv/data_migrations/VersionedSessions/sql
"""
use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
@suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
@versioned_table_engines [
"ReplicatedVersionedCollapsingMergeTree",
"VersionedCollapsingMergeTree"
]
def run(opts \\ []) do
run_exchange? = Keyword.get(opts, :run_exchange?, true)
unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
{:ok, %{rows: partitions}} = run_sql("list-partitions")
partitions = Enum.map(partitions, fn [part] -> part end)
{:ok, %{rows: [[current_table_engine, table_settings]]}} =
run_sql("get-sessions-table-settings")
if Enum.member?(@versioned_table_engines, current_table_engine) do
IO.puts("sessions_v2 table is already versioned, no migration needed")
else
{:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
{:ok, _} =
run_sql("create-sessions-tmp-table",
cluster?: cluster?,
table_settings: table_settings,
unique_suffix: unique_suffix
)
for partition <- partitions do
{:ok, _} = run_sql("attach-partition", partition: partition)
end
if run_exchange? do
run_exchange(cluster?)
end
IO.puts("Migration done!")
end
end
defp run_exchange(cluster?) do
case run_sql("exchange-sessions-tables", cluster?: cluster?) do
{:ok, _} ->
nil
# Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
{:error, %Ch.Error{code: 1}} ->
IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
{:ok, _} =
run_sql("rename-table",
from: "sessions_v2",
to: "sessions_v2_backup",
cluster?: cluster?
)
{:ok, _} =
run_sql("rename-table",
from: "sessions_v2_tmp_versioned",
to: "sessions_v2",
cluster?: cluster?
)
end
end
end

@ -4,9 +4,13 @@ defmodule Plausible.MigrationUtils do
"""
def on_cluster_statement(table) do
if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")
end
def clustered_table?(table) do
case Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do
{:ok, %{rows: []}} -> ""
{:ok, _} -> "ON CLUSTER '{cluster}'"
{:ok, %{rows: []}} -> false
{:ok, _} -> true
end
end
end

@ -0,0 +1 @@
ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2

@ -0,0 +1 @@
SELECT 1 FROM system.replicas WHERE table = 'sessions_v2'

@ -0,0 +1,13 @@
CREATE TABLE sessions_v2_tmp_versioned
<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
AS sessions_v2
<%= if @cluster? do %>
ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events)
<% else %>
ENGINE = VersionedCollapsingMergeTree(sign, events)
<% end %>
PARTITION BY toYYYYMM(start)
PRIMARY KEY (site_id, toDate(start), user_id, session_id)
ORDER BY (site_id, toDate(start), user_id, session_id)
SAMPLE BY user_id
<%= @table_settings %>

@ -0,0 +1 @@
DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>

@ -0,0 +1 @@
EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>

@ -0,0 +1,4 @@
SELECT engine, extract(engine_full, 'SETTINGS .+')
FROM system.tables
WHERE database = currentDatabase()
AND name = 'sessions_v2'

@ -0,0 +1,5 @@
SELECT distinct(partition)
FROM system.parts
WHERE database = currentDatabase()
AND table = 'sessions_v2'
ORDER BY partition

@ -0,0 +1 @@
RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
Loading…
Cancel
Save