diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 003689b0c..540b8a47c 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -70,12 +70,8 @@ jobs: run: npm run deploy --prefix ./tracker - name: Check Credo Warnings run: mix credo diff --from-git-merge-base origin/master - - name: Run tests (against v1 schema) + - name: Run tests run: mix test --include slow --max-failures 1 --warnings-as-errors - - name: Run tests (against v2 schema) - run: mix test --include slow --max-failures 1 --warnings-as-errors - env: - V2_MIGRATION_DONE: 1 - name: Check Dialyzer run: mix dialyzer env: diff --git a/config/.env.dev b/config/.env.dev index a42c5a0f8..771e90163 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -17,4 +17,3 @@ GOOGLE_CLIENT_SECRET=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 PROMEX_DISABLED=false SITE_DEFAULT_INGEST_THRESHOLD=1000000 -V2_MIGRATION_DONE=1 diff --git a/config/runtime.exs b/config/runtime.exs index e9b86ac08..82d8ceb6d 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -99,8 +99,6 @@ ch_db_url = |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000") |> Integer.parse() -v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE") - ### Mandatory params End build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}") @@ -133,7 +131,7 @@ runtime_metadata = [ config :plausible, :runtime_metadata, runtime_metadata -config :plausible, :v2_migration_done, v2_migration_done +config :plausible, :v2_migration_done, true sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN") honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY") diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex index be3885928..7655f56e3 100644 --- a/lib/plausible/data_migration.ex +++ b/lib/plausible/data_migration.ex @@ -1,6 +1,6 @@ defmodule Plausible.DataMigration do @moduledoc """ - Base module for coordinated Clickhouse data migrations + Base module for coordinated Clickhouse data migrations run via remote shell or otherwise (TBD). """ @@ -12,20 +12,43 @@ defmodule Plausible.DataMigration do @dir dir @repo repo - def run_sql_confirm(name, assigns \\ []) do - query = unwrap_with_io(name, assigns) + def run_sql_confirm(name, options \\ []) do + {prompt_options, assigns} = + Keyword.split(options, [:prompt_message, :prompt_default_choice]) - confirm("Execute?", fn -> do_run(name, query) end) + query = unwrap_with_io(name, assigns) + message = prompt_options[:prompt_message] || "Execute?" + default_choice = Keyword.get(prompt_options, :prompt_default_choice, :yes) + confirm(message, fn -> do_run(name, query) end, default_choice) end - def confirm(message, func) do - prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset() + def confirm(message, func, default_choice \\ :yes) do + choices = + case default_choice do + :yes -> " [Y/n]: " + :no -> " [y/N]: " + end - if String.downcase(String.trim(IO.gets(prompt))) == "n" do + prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset() + answer = String.downcase(String.trim(IO.gets(prompt))) + + skip = fn -> IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}") {:ok, :skip} - else - func.() + end + + case answer do + "y" -> + func.() + + "n" -> + skip.() + + _ -> + case default_choice do + :yes -> func.() + :no -> skip.() + end end end diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex index 004f8ca45..f8e859f0f 100644 --- a/lib/plausible/data_migration/numeric_ids.ex +++ b/lib/plausible/data_migration/numeric_ids.ex @@ -62,10 +62,9 @@ defmodule Plausible.DataMigration.NumericIDs do start_from = start_from || List.first(partitions) IO.puts(""" - Got the following migration settings: + Got the following migration settings: - max_threads: #{max_threads} - - dict_password: ✅ - table_settings: #{table_settings} - db url: #{db_url} - cluster?: #{cluster?} @@ -90,8 +89,28 @@ defmodule Plausible.DataMigration.NumericIDs do end end - {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?) - {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?) + drop_v2_extra_opts = fn table -> + case @repo.query("select count(*) from {$0:Identifier}", [table]) do + {:ok, %{rows: [[count]]}} when count > 0 -> + [ + prompt_message: "The table contains #{count} rows. Execute?", + prompt_default_choice: :no + ] + + {:ok, _} -> + [prompt_default_choice: :no] + + {:error, _} -> + [] + end + end + + {:ok, _} = + run_sql_fn.("drop-events-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("events_v2")) + + {:ok, _} = + run_sql_fn.("drop-sessions-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("sessions_v2")) + {:ok, _} = run_sql_fn.("drop-tmp-events-v2", []) {:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", []) {:ok, _} = run_sql_fn.("drop-domains-lookup", []) diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex index c5fd79b4d..26500f99f 100644 --- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex @@ -10,7 +10,7 @@ INSERT INTO tmp_events_v2 ( transferred_from ) SELECT - (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1), + site_id, timestamp, name, user_id, @@ -38,4 +38,5 @@ SELECT transferred_from FROM events +INNER JOIN domains_lookup ON domains_lookup.domain = events.domain WHERE (_partition_id = '<%= @partition %>') diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex index c50ca8cbd..e2222b144 100644 --- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex @@ -6,7 +6,7 @@ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_term, transferred_from, entry_meta.key, entry_meta.value ) SELECT - (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1), + site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page, exit_page, pageviews, events, duration, referrer, referrer_source, country_code, screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version, @@ -14,4 +14,5 @@ SELECT utm_term, transferred_from, entry_meta.key, entry_meta.value FROM sessions +INNER JOIN domains_lookup ON domains_lookup.domain = sessions.domain WHERE (_partition_id = '<%= @partition %>')