Make self-hosted data migration easier (#2865)

* default to v2

* allow N defaults in data migration prompt and custom messages

* join domains lookup

* remove duplicate test runs from ci (both are v2)
tagged-events-bug-repro
ruslandoga 4 years ago committed by GitHub
parent 586bbeb8ac
commit adcce15632
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -70,12 +70,8 @@ jobs:
run: npm run deploy --prefix ./tracker
- name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- name: Run tests (against v1 schema)
- name: Run tests
run: mix test --include slow --max-failures 1 --warnings-as-errors
- name: Run tests (against v2 schema)
run: mix test --include slow --max-failures 1 --warnings-as-errors
env:
V2_MIGRATION_DONE: 1
- name: Check Dialyzer
run: mix dialyzer
env:

@ -17,4 +17,3 @@ GOOGLE_CLIENT_SECRET=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
PROMEX_DISABLED=false
SITE_DEFAULT_INGEST_THRESHOLD=1000000
V2_MIGRATION_DONE=1

@ -99,8 +99,6 @@ ch_db_url =
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
|> Integer.parse()
v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")
### Mandatory params End
build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")
@ -133,7 +131,7 @@ runtime_metadata = [
config :plausible, :runtime_metadata, runtime_metadata
config :plausible, :v2_migration_done, v2_migration_done
config :plausible, :v2_migration_done, true
sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")
honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")

@ -1,6 +1,6 @@
defmodule Plausible.DataMigration do
@moduledoc """
Base module for coordinated Clickhouse data migrations
Base module for coordinated Clickhouse data migrations
run via remote shell or otherwise (TBD).
"""
@ -12,20 +12,43 @@ defmodule Plausible.DataMigration do
@dir dir
@repo repo
def run_sql_confirm(name, assigns \\ []) do
query = unwrap_with_io(name, assigns)
def run_sql_confirm(name, options \\ []) do
{prompt_options, assigns} =
Keyword.split(options, [:prompt_message, :prompt_default_choice])
confirm("Execute?", fn -> do_run(name, query) end)
query = unwrap_with_io(name, assigns)
message = prompt_options[:prompt_message] || "Execute?"
default_choice = Keyword.get(prompt_options, :prompt_default_choice, :yes)
confirm(message, fn -> do_run(name, query) end, default_choice)
end
def confirm(message, func) do
prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()
def confirm(message, func, default_choice \\ :yes) do
choices =
case default_choice do
:yes -> " [Y/n]: "
:no -> " [y/N]: "
end
if String.downcase(String.trim(IO.gets(prompt))) == "n" do
prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset()
answer = String.downcase(String.trim(IO.gets(prompt)))
skip = fn ->
IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")
{:ok, :skip}
else
func.()
end
case answer do
"y" ->
func.()
"n" ->
skip.()
_ ->
case default_choice do
:yes -> func.()
:no -> skip.()
end
end
end

@ -62,10 +62,9 @@ defmodule Plausible.DataMigration.NumericIDs do
start_from = start_from || List.first(partitions)
IO.puts("""
Got the following migration settings:
Got the following migration settings:
- max_threads: #{max_threads}
- dict_password: ✅
- table_settings: #{table_settings}
- db url: #{db_url}
- cluster?: #{cluster?}
@ -90,8 +89,28 @@ defmodule Plausible.DataMigration.NumericIDs do
end
end
{:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)
{:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)
drop_v2_extra_opts = fn table ->
case @repo.query("select count(*) from {$0:Identifier}", [table]) do
{:ok, %{rows: [[count]]}} when count > 0 ->
[
prompt_message: "The table contains #{count} rows. Execute?",
prompt_default_choice: :no
]
{:ok, _} ->
[prompt_default_choice: :no]
{:error, _} ->
[]
end
end
{:ok, _} =
run_sql_fn.("drop-events-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("events_v2"))
{:ok, _} =
run_sql_fn.("drop-sessions-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("sessions_v2"))
{:ok, _} = run_sql_fn.("drop-tmp-events-v2", [])
{:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])
{:ok, _} = run_sql_fn.("drop-domains-lookup", [])

@ -10,7 +10,7 @@ INSERT INTO tmp_events_v2 (
transferred_from
)
SELECT
(SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),
site_id,
timestamp,
name,
user_id,
@ -38,4 +38,5 @@ SELECT
transferred_from
FROM
events
INNER JOIN domains_lookup ON domains_lookup.domain = events.domain
WHERE (_partition_id = '<%= @partition %>')

@ -6,7 +6,7 @@ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id,
utm_term, transferred_from, entry_meta.key, entry_meta.value
)
SELECT
(SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),
site_id,
session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
@ -14,4 +14,5 @@ SELECT
utm_term, transferred_from, entry_meta.key, entry_meta.value
FROM
sessions
INNER JOIN domains_lookup ON domains_lookup.domain = sessions.domain
WHERE (_partition_id = '<%= @partition %>')

Loading…
Cancel
Save