From 1087742feaefad025ed204d00425488dc4e39c95 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 21 May 2020 15:55:46 +0300 Subject: [PATCH] Clean after itself --- lib/mix/tasks/hydrate_clickhouse.ex | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/hydrate_clickhouse.ex b/lib/mix/tasks/hydrate_clickhouse.ex index 6e3ef6f6f..a8f9776bc 100644 --- a/lib/mix/tasks/hydrate_clickhouse.ex +++ b/lib/mix/tasks/hydrate_clickhouse.ex @@ -105,7 +105,7 @@ defmodule Mix.Tasks.HydrateClickhouse do e in Plausible.Event, where: e.timestamp < ^end_time, order_by: e.id - ) |> chunk_query(10_000, repo) + ) |> chunk_query(50_000, repo) Enum.reduce(event_chunks, {%{}, 0}, fn events, {session_cache, processed_events} -> {session_cache, sessions, events} = Enum.reduce(events, {session_cache, [], []}, fn event, {session_cache, sessions, new_events} -> @@ -141,12 +141,29 @@ defmodule Mix.Tasks.HydrateClickhouse do Plausible.Clickhouse.insert_events(events) Plausible.Clickhouse.insert_sessions(sessions) + session_cache = clean(session_cache, List.last(events).timestamp) new_processed_count = processed_events + Enum.count(events) IO.puts("Processed #{new_processed_count} out of #{total} (#{round(new_processed_count / total * 100)}%)") {session_cache, processed_events + Enum.count(events)} end) end + defp clean(session_cache, latest_timestamp) do + cleaned = Enum.reduce(session_cache, %{}, fn {key, session}, acc -> + if Timex.diff(latest_timestamp, session.timestamp, :second) <= 3600 do + Map.put(acc, key, session) + else + acc # forget the session + end + end) + + n_old = Enum.count(session_cache) + n_new = Enum.count(cleaned) + + IO.puts("Removed #{n_old - n_new} sessions from store") + cleaned + end + defp is_active?(session, event) do session && Timex.diff(event.timestamp, session.timestamp, :minute) <= 29 end