Implement Numeric IDs migration (#2762)
* Implement Numeric IDs migration * Fix typo * Mute credo for now * Improve configurability and add stop_t * Adjust to Ch/Chto only * Fix opts key for dictionary password * Add regular ecto migration with numeric ids v2 schemas (#2768) * Add regular ecto migration * Fix typo * Update priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs Co-authored-by: Vini Brasil <vini@hey.com> * Implement v2 events/sessions schema modules (#2777) * Implement v2 events/sessions schema modules * Clean up session schemas --------- Co-authored-by: Vini Brasil <vini@hey.com> * Update moduledocs --------- Co-authored-by: Vini Brasil <vini@hey.com>pull/2787/head r2023.03.23-82
parent
a129bf82e0
commit
6637751a5e
@ -0,0 +1,76 @@
|
||||
defmodule Plausible.ClickhouseEventV2 do
|
||||
@moduledoc """
|
||||
Event schema for when NumericIDs migration is complete
|
||||
"""
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
@primary_key false
|
||||
schema "events_v2" do
|
||||
field :name, :string
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :hostname, :string
|
||||
field :pathname, :string
|
||||
field :user_id, Ch.Types.UInt64
|
||||
field :session_id, Ch.Types.UInt64
|
||||
field :timestamp, :naive_datetime
|
||||
|
||||
field :referrer, :string
|
||||
field :referrer_source, :string
|
||||
field :utm_medium, :string
|
||||
field :utm_source, :string
|
||||
field :utm_campaign, :string
|
||||
field :utm_content, :string
|
||||
field :utm_term, :string
|
||||
|
||||
field :country_code, Ch.Types.FixedString, size: 2
|
||||
field :subdivision1_code, :string
|
||||
field :subdivision2_code, :string
|
||||
field :city_geoname_id, Ch.Types.UInt32
|
||||
|
||||
field :screen_size, :string
|
||||
field :operating_system, :string
|
||||
field :operating_system_version, :string
|
||||
field :browser, :string
|
||||
field :browser_version, :string
|
||||
|
||||
field :"meta.key", {:array, :string}
|
||||
field :"meta.value", {:array, :string}
|
||||
field :transferred_from, :string
|
||||
end
|
||||
|
||||
def new(attrs) do
|
||||
%__MODULE__{}
|
||||
|> cast(
|
||||
attrs,
|
||||
[
|
||||
:name,
|
||||
:site_id,
|
||||
:hostname,
|
||||
:pathname,
|
||||
:user_id,
|
||||
:timestamp,
|
||||
:operating_system,
|
||||
:operating_system_version,
|
||||
:browser,
|
||||
:browser_version,
|
||||
:referrer,
|
||||
:referrer_source,
|
||||
:utm_medium,
|
||||
:utm_source,
|
||||
:utm_campaign,
|
||||
:utm_content,
|
||||
:utm_term,
|
||||
:country_code,
|
||||
:subdivision1_code,
|
||||
:subdivision2_code,
|
||||
:city_geoname_id,
|
||||
:screen_size,
|
||||
:"meta.key",
|
||||
:"meta.value"
|
||||
],
|
||||
empty_values: [nil, ""]
|
||||
)
|
||||
|> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,52 @@
|
||||
defmodule Plausible.ClickhouseSessionV2 do
|
||||
@moduledoc """
|
||||
Session schema for when NumericIDs migration is complete
|
||||
"""
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "sessions_v2" do
|
||||
field :hostname, :string
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :user_id, Ch.Types.UInt64
|
||||
field :session_id, Ch.Types.UInt64
|
||||
|
||||
field :start, :naive_datetime
|
||||
field :duration, Ch.Types.UInt32
|
||||
field :is_bounce, :boolean
|
||||
field :entry_page, :string
|
||||
field :exit_page, :string
|
||||
field :pageviews, Ch.Types.Int32
|
||||
field :events, Ch.Types.Int32
|
||||
field :sign, Ch.Types.Int8
|
||||
|
||||
field :"entry_meta.key", {:array, :string}
|
||||
field :"entry_meta.value", {:array, :string}
|
||||
|
||||
field :utm_medium, :string
|
||||
field :utm_source, :string
|
||||
field :utm_campaign, :string
|
||||
field :utm_content, :string
|
||||
field :utm_term, :string
|
||||
field :referrer, :string
|
||||
field :referrer_source, :string
|
||||
|
||||
field :country_code, Ch.Types.FixedString, size: 2
|
||||
field :subdivision1_code, :string
|
||||
field :subdivision2_code, :string
|
||||
field :city_geoname_id, Ch.Types.UInt32
|
||||
|
||||
field :screen_size, :string
|
||||
field :operating_system, :string
|
||||
field :operating_system_version, :string
|
||||
field :browser, :string
|
||||
field :browser_version, :string
|
||||
field :timestamp, :naive_datetime
|
||||
|
||||
field :transferred_from, :string
|
||||
end
|
||||
|
||||
def random_uint64() do
|
||||
:crypto.strong_rand_bytes(8) |> :binary.decode_unsigned()
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,64 @@
|
||||
defmodule Plausible.DataMigration do
|
||||
@moduledoc """
|
||||
Base module for coordinated Clickhouse data migrations
|
||||
run via remote shell or otherwise (TBD).
|
||||
"""
|
||||
|
||||
defmacro __using__(opts) do
|
||||
dir = Keyword.fetch!(opts, :dir)
|
||||
repo = Keyword.get(opts, :repo, Plausible.DataMigration.Repo)
|
||||
|
||||
quote bind_quoted: [dir: dir, repo: repo] do
|
||||
@dir dir
|
||||
@repo repo
|
||||
|
||||
def run_sql_confirm(name, assigns \\ []) do
|
||||
query = unwrap(name, assigns)
|
||||
|
||||
confirm("Execute?", fn -> do_run(name, query) end)
|
||||
end
|
||||
|
||||
def confirm(message, func) do
|
||||
prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()
|
||||
|
||||
if String.downcase(String.trim(IO.gets(prompt))) == "n" do
|
||||
IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")
|
||||
{:ok, :skip}
|
||||
else
|
||||
func.()
|
||||
end
|
||||
end
|
||||
|
||||
defp unwrap(name, assigns) do
|
||||
"priv/data_migrations"
|
||||
|> Path.join(@dir)
|
||||
|> Path.join("sql")
|
||||
|> Path.join(name <> ".sql.eex")
|
||||
|> EEx.eval_file(assigns: assigns)
|
||||
end
|
||||
|
||||
def run_sql(name, assigns \\ []) do
|
||||
query = unwrap(name, assigns)
|
||||
do_run(name, query)
|
||||
end
|
||||
|
||||
defp do_run(name, query) do
|
||||
{:ok, res} = @repo.query(query, [], timeout: :infinity)
|
||||
IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
|
||||
IO.puts(String.duplicate("-", 78))
|
||||
{:ok, res}
|
||||
end
|
||||
|
||||
defp unwrap_with_io(name, assigns) do
|
||||
IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}")
|
||||
query = unwrap(name, assigns)
|
||||
|
||||
IO.puts("""
|
||||
-> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()}
|
||||
""")
|
||||
|
||||
query
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,141 @@
|
||||
defmodule Plausible.DataMigration.NumericIDs do
|
||||
@moduledoc """
|
||||
Numeric IDs migration, SQL files available at:
|
||||
priv/data_migrations/NumericIDs/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "NumericIDs"
|
||||
|
||||
@table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"
|
||||
|
||||
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
|
||||
def run(opts \\ []) do
|
||||
interactive? = Keyword.get(opts, :interactive?, true)
|
||||
|
||||
db_url =
|
||||
System.get_env(
|
||||
"NUMERIC_IDS_MIGRATION_DB_URL",
|
||||
Application.get_env(:plausible, Plausible.IngestRepo)[:url]
|
||||
)
|
||||
|
||||
max_threads =
|
||||
"NUMERIC_IDS_MIGRATION_MAX_THREADS" |> System.get_env("16") |> String.to_integer()
|
||||
|
||||
# TBD: There's most likely a bug in Clickhouse defining Postgres dictionaries,
|
||||
# we'll use a static URL for now
|
||||
dict_url = Keyword.get(opts, :dict_url) || System.get_env("DOMAINS_DICT_URL") || ""
|
||||
|
||||
dict_password =
|
||||
Keyword.get(opts, :dict_password) || System.get_env("DOMAINS_DICT_PASSWORD") || ""
|
||||
|
||||
table_settings =
|
||||
Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||
|
||||
@table_settings
|
||||
|
||||
start_from =
|
||||
Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")
|
||||
|
||||
stop_at =
|
||||
Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT") ||
|
||||
previous_part()
|
||||
|
||||
(byte_size(dict_url) > 0 and byte_size(dict_password) > 0) ||
|
||||
raise "Set DOMAINS_DICT_URL and DOMAINS_DICT_PASSWORD"
|
||||
|
||||
@repo.start(db_url, max_threads)
|
||||
|
||||
cluster? =
|
||||
case run_sql("check-replicas") do
|
||||
{:ok, %{num_rows: 0}} -> false
|
||||
{:ok, %{num_rows: 1}} -> true
|
||||
end
|
||||
|
||||
{:ok, %{rows: partitions}} =
|
||||
run_sql("list-partitions", start_from: start_from, stop_at: stop_at)
|
||||
|
||||
partitions = Enum.map(partitions, fn [part] -> part end)
|
||||
|
||||
start_from = start_from || List.first(partitions)
|
||||
|
||||
IO.puts("""
|
||||
Got the following migration settings:
|
||||
|
||||
- max_threads: #{max_threads}
|
||||
- dict_url: #{dict_url}
|
||||
- dict_password: ✅
|
||||
- table_settings: #{table_settings}
|
||||
- db url: #{db_url}
|
||||
- cluster?: #{cluster?}
|
||||
- partitions to do: #{inspect(partitions, pretty: true, width: 80)}
|
||||
- start from: #{start_from}
|
||||
- stop at: #{stop_at}
|
||||
""")
|
||||
|
||||
run_sql_fn =
|
||||
if interactive? do
|
||||
&run_sql_confirm/2
|
||||
else
|
||||
&run_sql/2
|
||||
end
|
||||
|
||||
confirm_fn =
|
||||
if interactive? do
|
||||
&confirm/2
|
||||
else
|
||||
fn _, run_fn ->
|
||||
run_fn.()
|
||||
end
|
||||
end
|
||||
|
||||
{:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)
|
||||
{:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)
|
||||
{:ok, _} = run_sql_fn.("drop-tmp-events-v2", [])
|
||||
{:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])
|
||||
{:ok, _} = run_sql_fn.("drop-dict", [])
|
||||
|
||||
{:ok, _} =
|
||||
run_sql_fn.("create-dict-from-static-file",
|
||||
dict_url: dict_url,
|
||||
dict_password: dict_password
|
||||
)
|
||||
|
||||
{:ok, _} = run_sql_fn.("create-events-v2", table_settings: table_settings, cluster?: cluster?)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql_fn.("create-sessions-v2", table_settings: table_settings, cluster?: cluster?)
|
||||
|
||||
{:ok, _} = run_sql_fn.("create-tmp-events-v2", table_settings: table_settings)
|
||||
{:ok, _} = run_sql_fn.("create-tmp-sessions-v2", table_settings: table_settings)
|
||||
|
||||
confirm_fn.("Start migration? (starting from partition: #{start_from})", fn ->
|
||||
IO.puts("start.. #{DateTime.utc_now()}")
|
||||
|
||||
for part <- partitions do
|
||||
part_start = System.monotonic_time()
|
||||
|
||||
confirm_fn.("Run partition: #{part}?", fn ->
|
||||
{:ok, _} = run_sql("insert-into-tmp-events-v2", partition: part)
|
||||
{:ok, _} = run_sql("attach-tmp-events-v2", partition: part)
|
||||
{:ok, _} = run_sql("truncate-tmp-events-v2", [])
|
||||
{:ok, _} = run_sql("insert-into-tmp-sessions-v2", partition: part)
|
||||
{:ok, _} = run_sql("attach-tmp-sessions-v2", partition: part)
|
||||
{:ok, _} = run_sql("truncate-tmp-sessions-v2", [])
|
||||
end)
|
||||
|
||||
part_end = System.monotonic_time()
|
||||
|
||||
IO.puts(
|
||||
"#{part} took #{System.convert_time_unit(part_end - part_start, :native, :second)} seconds"
|
||||
)
|
||||
end
|
||||
|
||||
IO.puts("end.. #{DateTime.utc_now()}")
|
||||
end)
|
||||
end
|
||||
|
||||
defp previous_part() do
|
||||
now = NaiveDateTime.utc_now()
|
||||
month = String.pad_leading("#{now.month - 1}", 2, "0")
|
||||
year = "#{now.year}"
|
||||
"#{year}#{month}"
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,22 @@
|
||||
defmodule Plausible.DataMigration.Repo do
|
||||
@moduledoc """
|
||||
Ecto.Repo for Clickhouse data migrations, to be started manually,
|
||||
outside of the main application supervision tree.
|
||||
"""
|
||||
use Ecto.Repo,
|
||||
otp_app: :plausible,
|
||||
adapter: Ecto.Adapters.ClickHouse
|
||||
|
||||
def start(url, max_threads) when is_binary(url) and is_integer(max_threads) do
|
||||
start_link(
|
||||
url: url,
|
||||
queue_target: 500,
|
||||
queue_interval: 2000,
|
||||
pool_size: 1,
|
||||
settings: [
|
||||
max_insert_threads: max_threads,
|
||||
send_progress_in_http_headers: 1
|
||||
]
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1 @@
|
||||
ALTER TABLE events_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_events_v2
|
||||
@ -0,0 +1 @@
|
||||
ALTER TABLE sessions_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_sessions_v2
|
||||
@ -0,0 +1 @@
|
||||
SELECT 1 FROM system.replicas WHERE table = 'events'
|
||||
@ -0,0 +1,9 @@
|
||||
CREATE DICTIONARY domains_lookup (
|
||||
`id` UInt64,
|
||||
`site_id` UInt64,
|
||||
`domain` String,
|
||||
`partition` String
|
||||
) PRIMARY KEY domain
|
||||
SOURCE(HTTP(URL '<%= @dict_url %>' CREDENTIALS(user 'user' password '<%= @dict_password %>') FORMAT 'CSVWithNames'))
|
||||
LIFETIME(3600)
|
||||
LAYOUT(COMPLEX_KEY_HASHED())
|
||||
@ -0,0 +1,39 @@
|
||||
CREATE TABLE events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
(
|
||||
`timestamp` DateTime CODEC(Delta(4), LZ4),
|
||||
`name` LowCardinality(String),
|
||||
`site_id` UInt64,
|
||||
`user_id` UInt64,
|
||||
`session_id` UInt64,
|
||||
`hostname` String,
|
||||
`pathname` String CODEC(ZSTD(3)),
|
||||
`referrer` String,
|
||||
`referrer_source` String,
|
||||
`country_code` FixedString(2),
|
||||
`screen_size` LowCardinality(String),
|
||||
`operating_system` LowCardinality(String),
|
||||
`browser` LowCardinality(String),
|
||||
`utm_medium` String,
|
||||
`utm_source` String,
|
||||
`utm_campaign` String,
|
||||
`meta.key` Array(String),
|
||||
`meta.value` Array(String),
|
||||
`browser_version` LowCardinality(String),
|
||||
`operating_system_version` LowCardinality(String),
|
||||
`subdivision1_code` LowCardinality(String),
|
||||
`subdivision2_code` LowCardinality(String),
|
||||
`city_geoname_id` UInt32,
|
||||
`utm_content` String,
|
||||
`utm_term` String,
|
||||
`transferred_from` String
|
||||
)
|
||||
<%= if @cluster? do %>
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/events_v2', '{replica}')
|
||||
<% else %>
|
||||
ENGINE = MergeTree()
|
||||
<% end %>
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
PRIMARY KEY (site_id, toDate(timestamp), name, user_id)
|
||||
ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)
|
||||
SAMPLE BY user_id
|
||||
<%= @table_settings %>
|
||||
@ -0,0 +1,45 @@
|
||||
CREATE TABLE sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
|
||||
(
|
||||
`session_id` UInt64,
|
||||
`sign` Int8,
|
||||
`site_id` UInt64,
|
||||
`user_id` UInt64,
|
||||
`hostname` String,
|
||||
`timestamp` DateTime CODEC(DoubleDelta, LZ4),
|
||||
`start` DateTime CODEC(DoubleDelta, LZ4),
|
||||
`is_bounce` UInt8,
|
||||
`entry_page` String,
|
||||
`exit_page` String,
|
||||
`pageviews` Int32,
|
||||
`events` Int32,
|
||||
`duration` UInt32,
|
||||
`referrer` String,
|
||||
`referrer_source` String,
|
||||
`country_code` LowCardinality(FixedString(2)),
|
||||
`screen_size` LowCardinality(String),
|
||||
`operating_system` LowCardinality(String),
|
||||
`browser` LowCardinality(String),
|
||||
`utm_medium` String,
|
||||
`utm_source` String,
|
||||
`utm_campaign` String,
|
||||
`browser_version` LowCardinality(String),
|
||||
`operating_system_version` LowCardinality(String),
|
||||
`subdivision1_code` LowCardinality(String),
|
||||
`subdivision2_code` LowCardinality(String),
|
||||
`city_geoname_id` UInt32,
|
||||
`utm_content` String,
|
||||
`utm_term` String,
|
||||
`transferred_from` String,
|
||||
`entry_meta.key` Array(String),
|
||||
`entry_meta.value` Array(String)
|
||||
)
|
||||
<%= if @cluster? do %>
|
||||
ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2', '{replica}', sign)
|
||||
<% else %>
|
||||
ENGINE = CollapsingMergeTree(sign)
|
||||
<% end %>
|
||||
PARTITION BY toYYYYMM(start)
|
||||
PRIMARY KEY (site_id, toDate(start), user_id, session_id)
|
||||
ORDER BY (site_id, toDate(start), user_id, session_id)
|
||||
SAMPLE BY user_id
|
||||
<%= @table_settings %>
|
||||
@ -0,0 +1,6 @@
|
||||
CREATE TABLE IF NOT EXISTS
|
||||
tmp_events_v2 AS events_v2 ENGINE = MergeTree()
|
||||
ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)
|
||||
PRIMARY KEY (site_id, toDate(timestamp), name, user_id)
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
<%= @table_settings %>
|
||||
@ -0,0 +1,6 @@
|
||||
CREATE TABLE IF NOT EXISTS
|
||||
tmp_sessions_v2 AS sessions_v2 ENGINE = CollapsingMergeTree(sign)
|
||||
PARTITION BY toYYYYMM(start)
|
||||
PRIMARY KEY (site_id, toDate(start), user_id, session_id)
|
||||
ORDER BY (site_id, toDate(start), user_id, session_id)
|
||||
<%= @table_settings %>
|
||||
@ -0,0 +1 @@
|
||||
DROP DICTIONARY IF EXISTS domains_lookup SYNC
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS tmp_events_v2 SYNC
|
||||
@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS tmp_sessions_v2 SYNC
|
||||
@ -0,0 +1,54 @@
|
||||
INSERT INTO tmp_events_v2 (
|
||||
site_id, timestamp, name, user_id,
|
||||
session_id, hostname, pathname, referrer,
|
||||
referrer_source, country_code, screen_size,
|
||||
operating_system, browser, utm_medium,
|
||||
utm_source, utm_campaign, meta.key,
|
||||
meta.value, browser_version, operating_system_version,
|
||||
subdivision1_code, subdivision2_code,
|
||||
city_geoname_id, utm_content, utm_term,
|
||||
transferred_from
|
||||
)
|
||||
SELECT
|
||||
dictGet(
|
||||
'domains_lookup', 'site_id', domain
|
||||
),
|
||||
timestamp,
|
||||
name,
|
||||
user_id,
|
||||
session_id,
|
||||
hostname,
|
||||
pathname,
|
||||
referrer,
|
||||
referrer_source,
|
||||
country_code,
|
||||
screen_size,
|
||||
operating_system,
|
||||
browser,
|
||||
utm_medium,
|
||||
utm_source,
|
||||
utm_campaign,
|
||||
meta.key,
|
||||
meta.value,
|
||||
browser_version,
|
||||
operating_system_version,
|
||||
subdivision1_code,
|
||||
subdivision2_code,
|
||||
city_geoname_id,
|
||||
utm_content,
|
||||
utm_term,
|
||||
transferred_from
|
||||
FROM
|
||||
events
|
||||
WHERE
|
||||
(
|
||||
domain IN (
|
||||
SELECT
|
||||
domain
|
||||
FROM
|
||||
dictionary('domains_lookup')
|
||||
WHERE
|
||||
partition <= '<%= @partition %>'
|
||||
)
|
||||
)
|
||||
AND (_partition_id = '<%= @partition %>')
|
||||
@ -0,0 +1,30 @@
|
||||
INSERT INTO tmp_sessions_v2 (
|
||||
site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
|
||||
exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
|
||||
screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
|
||||
operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,
|
||||
utm_term, transferred_from, entry_meta.key, entry_meta.value
|
||||
)
|
||||
SELECT
|
||||
dictGet(
|
||||
'domains_lookup', 'site_id', domain
|
||||
),
|
||||
session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
|
||||
exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
|
||||
screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
|
||||
operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,
|
||||
utm_term, transferred_from, entry_meta.key, entry_meta.value
|
||||
FROM
|
||||
sessions
|
||||
WHERE
|
||||
(
|
||||
domain IN (
|
||||
SELECT
|
||||
domain
|
||||
FROM
|
||||
dictionary('domains_lookup')
|
||||
WHERE
|
||||
partition <= '<%= @partition %>'
|
||||
)
|
||||
)
|
||||
AND (_partition_id = '<%= @partition %>')
|
||||
@ -0,0 +1,6 @@
|
||||
SELECT partition
|
||||
FROM system.parts
|
||||
WHERE table = 'events'
|
||||
<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>
|
||||
<%= if @stop_at do %>AND partition <= '<%= @stop_at %>'<% end %>
|
||||
ORDER BY partition
|
||||
@ -0,0 +1 @@
|
||||
TRUNCATE TABLE tmp_events_v2
|
||||
@ -0,0 +1 @@
|
||||
TRUNCATE TABLE tmp_sessions_v2
|
||||
@ -0,0 +1,33 @@
|
||||
defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do
|
||||
@moduledoc """
|
||||
Normally, for live environments the migration will be done via
|
||||
`DataMigration.NumericIDs` module (TBD). In which case PASS_V2_SCHEMA_MIGRATION
|
||||
environment variable needs to be set, to only make the standard migrate
|
||||
command write an entry into schema_migrations.
|
||||
|
||||
For tests, and entirely new small, self-hosted instances however,
|
||||
we want to keep the ability of preparing the database without enforcing
|
||||
any data migration.
|
||||
"""
|
||||
|
||||
use Ecto.Migration
|
||||
|
||||
use Plausible.DataMigration, dir: "NumericIDs"
|
||||
|
||||
@cluster? false
|
||||
@settings "SETTINGS index_granularity = 8192"
|
||||
|
||||
def up do
|
||||
if System.get_env("PASS_V2_SCHEMA_MIGRATION") do
|
||||
:ok
|
||||
else
|
||||
execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)
|
||||
execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)
|
||||
end
|
||||
end
|
||||
|
||||
def down do
|
||||
execute unwrap("drop-events-v2", cluster?: @cluster?)
|
||||
execute unwrap("drop-sessions-v2", cluster?: @cluster?)
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue