From af9995265eb00be2a91ab2c77dcfad189d1c6eb7 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 13:54:26 +0200 Subject: [PATCH 1/4] Create a worker to clean clickhouse deleted sites data The plan is to run this weekly, but going to trigger it manually the first few times on cloud --- lib/workers/clickhouse_clean_sites.ex | 63 ++++++++++++++++++ test/workers/clickhouse_clean_sites_test.exs | 69 ++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 lib/workers/clickhouse_clean_sites.ex create mode 100644 test/workers/clickhouse_clean_sites_test.exs diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex new file mode 100644 index 000000000..62541c91f --- /dev/null +++ b/lib/workers/clickhouse_clean_sites.ex @@ -0,0 +1,63 @@ +defmodule Plausible.Workers.ClickhouseCleanSites do + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.IngestRepo + use Oban.Worker, queue: :clickhouse_clean_sites + + import Ecto.Query + + require Logger + + @tables_to_clear [ + "events_v2", + "sessions_v2", + "ingest_counters", + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_pages", + "imported_sources", + "imported_visitors" + ] + + def perform(_job) do + deleted_sites = get_deleted_sites_with_clickhouse_data() + + if length(deleted_sites) > 0 do + Logger.info( + "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" + ) + + site_ids_expr = deleted_sites |> Enum.map(&to_string/1) |> Enum.join(", ") + + for table <- @tables_to_clear do + IngestRepo.query( + "ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})", + [], + if(Mix.env() == :test, do: [mutations_sync: 1], else: []) + ) + end + end + + :ok + end + + def get_deleted_sites_with_clickhouse_data() do + pg_sites = + from(s in Plausible.Site, select: %{id: s.id}) + |> Plausible.Repo.all() + |> Enum.map(&(&1 |> Map.fetch!(:id))) + |> MapSet.new() + + ch_sites = + from(e in "events_v2", group_by: e.site_id, select: %{site_id: e.site_id}) + |> Plausible.ClickhouseRepo.all(timeout: :infinity) + |> Enum.map(& &1.site_id) + |> MapSet.new() + + MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list() + end +end diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs new file mode 100644 index 000000000..debb4c1e6 --- /dev/null +++ b/test/workers/clickhouse_clean_sites_test.exs @@ -0,0 +1,69 @@ +defmodule Plausible.Workers.ClickhouseCleanSitesTest do + use Plausible.DataCase + use Plausible.TestUtils + use Plausible + import Plausible.Factory + + alias Plausible.Workers.ClickhouseCleanSites + + test "deletes data from events and sessions tables" do + site = insert(:site) + deleted_site = insert(:site) + + populate_stats(site, [ + build(:pageview) + ]) + + populate_stats(deleted_site, [ + build(:pageview), + build(:pageview), + build(:imported_visitors), + build(:imported_sources), + build(:imported_pages), + build(:imported_entry_pages), + build(:imported_exit_pages), + build(:imported_locations), + build(:imported_devices), + build(:imported_browsers), + build(:imported_operating_systems) + ]) + + Repo.delete!(deleted_site) + + assert Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + site.id + ) + + ClickhouseCleanSites.perform(nil) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + + assert get_count("events_v2") == 1 + assert get_count("sessions_v2") == 1 + assert get_count("imported_visitors") == 0 + assert get_count("imported_sources") == 0 + assert get_count("imported_pages") == 0 + assert get_count("imported_entry_pages") == 0 + assert get_count("imported_exit_pages") == 0 + assert get_count("imported_locations") == 0 + assert get_count("imported_devices") == 0 + assert get_count("imported_browsers") == 0 + assert get_count("imported_operating_systems") == 0 + end + + def get_count(table) do + %{count: count} = + from(e in table, select: %{count: fragment("count()")}) |> Plausible.ClickhouseRepo.one() + + count + end +end From fbccc84fd7bfc77cc23d5e2b640b8951e277fe4b Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 14:42:28 +0200 Subject: [PATCH 2/4] Make asserting count more reliable --- lib/workers/clickhouse_clean_sites.ex | 2 +- test/workers/clickhouse_clean_sites_test.exs | 34 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex index 62541c91f..22881e193 100644 --- a/lib/workers/clickhouse_clean_sites.ex +++ b/lib/workers/clickhouse_clean_sites.ex @@ -37,7 +37,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do IngestRepo.query( "ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})", [], - if(Mix.env() == :test, do: [mutations_sync: 1], else: []) + if(Mix.env() == :test, do: [mutations_sync: 2], else: []) ) end end diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs index debb4c1e6..8d85dee2c 100644 --- a/test/workers/clickhouse_clean_sites_test.exs +++ b/test/workers/clickhouse_clean_sites_test.exs @@ -42,28 +42,28 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do ClickhouseCleanSites.perform(nil) + assert_count(deleted_site, "events_v2", 0) + assert_count(deleted_site, "sessions_v2", 0) + assert_count(deleted_site, "imported_visitors", 0) + assert_count(deleted_site, "imported_sources", 0) + assert_count(deleted_site, "imported_pages", 0) + assert_count(deleted_site, "imported_entry_pages", 0) + assert_count(deleted_site, "imported_exit_pages", 0) + assert_count(deleted_site, "imported_locations", 0) + assert_count(deleted_site, "imported_devices", 0) + assert_count(deleted_site, "imported_browsers", 0) + assert_count(deleted_site, "imported_operating_systems", 0) + assert_count(site, "events_v2", 1) + assert_count(site, "sessions_v2", 1) + assert not Enum.member?( ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), deleted_site.id ) - - assert get_count("events_v2") == 1 - assert get_count("sessions_v2") == 1 - assert get_count("imported_visitors") == 0 - assert get_count("imported_sources") == 0 - assert get_count("imported_pages") == 0 - assert get_count("imported_entry_pages") == 0 - assert get_count("imported_exit_pages") == 0 - assert get_count("imported_locations") == 0 - assert get_count("imported_devices") == 0 - assert get_count("imported_browsers") == 0 - assert get_count("imported_operating_systems") == 0 end - def get_count(table) do - %{count: count} = - from(e in table, select: %{count: fragment("count()")}) |> Plausible.ClickhouseRepo.one() - - count + def assert_count(site, table, expected_count) do + q = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id) + await_clickhouse_count(q, expected_count) end end From 120cd37c6c2f8b8732bbae4650258af8b2110d4b Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 28 Mar 2024 14:11:42 +0200 Subject: [PATCH 3/4] credo --- lib/workers/clickhouse_clean_sites.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex index 22881e193..3ab6502cf 100644 --- a/lib/workers/clickhouse_clean_sites.ex +++ b/lib/workers/clickhouse_clean_sites.ex @@ -1,4 +1,11 @@ defmodule Plausible.Workers.ClickhouseCleanSites do + @moduledoc """ + Cleans deleted site data from ClickHouse asynchronously. + + We batch up data deletions from ClickHouse as deleting a single site is + just as expensive as deleting many. + """ + use Plausible.Repo use Plausible.ClickhouseRepo use Plausible.IngestRepo @@ -31,7 +38,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" ) - site_ids_expr = deleted_sites |> Enum.map(&to_string/1) |> Enum.join(", ") + site_ids_expr = deleted_sites |> Enum.map_join(", ", &to_string/1) for table <- @tables_to_clear do IngestRepo.query( From 35be0cad1a427fd5c476710ce2535f665386eaa0 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 2 Apr 2024 09:22:15 +0300 Subject: [PATCH 4/4] PR feedback --- lib/workers/clickhouse_clean_sites.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex index 3ab6502cf..e29e2493f 100644 --- a/lib/workers/clickhouse_clean_sites.ex +++ b/lib/workers/clickhouse_clean_sites.ex @@ -30,6 +30,8 @@ defmodule Plausible.Workers.ClickhouseCleanSites do "imported_visitors" ] + @settings if Mix.env() in [:test, :small_test], do: [mutations_sync: 2], else: [] + def perform(_job) do deleted_sites = get_deleted_sites_with_clickhouse_data() @@ -41,10 +43,10 @@ defmodule Plausible.Workers.ClickhouseCleanSites do site_ids_expr = deleted_sites |> Enum.map_join(", ", &to_string/1) for table <- @tables_to_clear do - IngestRepo.query( - "ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})", + IngestRepo.query!( + "ALTER TABLE #{table} DELETE WHERE site_id IN #{site_ids_expr}", [], - if(Mix.env() == :test, do: [mutations_sync: 2], else: []) + settings: @settings ) end end @@ -54,15 +56,13 @@ defmodule Plausible.Workers.ClickhouseCleanSites do def get_deleted_sites_with_clickhouse_data() do pg_sites = - from(s in Plausible.Site, select: %{id: s.id}) + from(s in Plausible.Site, select: s.id) |> Plausible.Repo.all() - |> Enum.map(&(&1 |> Map.fetch!(:id))) |> MapSet.new() ch_sites = - from(e in "events_v2", group_by: e.site_id, select: %{site_id: e.site_id}) + from(e in "events_v2", group_by: e.site_id, select: e.site_id) |> Plausible.ClickhouseRepo.all(timeout: :infinity) - |> Enum.map(& &1.site_id) |> MapSet.new() MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()