diff --git a/lib/mix/tasks/hydrate_clickhouse.ex b/lib/mix/tasks/hydrate_clickhouse.ex index f1c7cbac8..6e3ef6f6f 100644 --- a/lib/mix/tasks/hydrate_clickhouse.ex +++ b/lib/mix/tasks/hydrate_clickhouse.ex @@ -99,13 +99,15 @@ defmodule Mix.Tasks.HydrateClickhouse do def hydrate_events(repo, _args \\ []) do end_time = ~N[2020-05-21 10:46:51] + total = Repo.aggregate(from(e in Plausible.Event, where: e.timestamp < ^end_time), :count, :id) + event_chunks = from( e in Plausible.Event, where: e.timestamp < ^end_time, order_by: e.id ) |> chunk_query(10_000, repo) - Enum.reduce(event_chunks, %{}, fn events, session_cache -> + Enum.reduce(event_chunks, {%{}, 0}, fn events, {session_cache, processed_events} -> {session_cache, sessions, events} = Enum.reduce(events, {session_cache, [], []}, fn event, {session_cache, sessions, new_events} -> found_session = session_cache[event.fingerprint] active = is_active?(found_session, event) @@ -139,7 +141,9 @@ defmodule Mix.Tasks.HydrateClickhouse do Plausible.Clickhouse.insert_events(events) Plausible.Clickhouse.insert_sessions(sessions) - session_cache + new_processed_count = processed_events + Enum.count(events) + IO.puts("Processed #{new_processed_count} out of #{total} (#{round(new_processed_count / total * 100)}%)") + {session_cache, processed_events + Enum.count(events)} end) end