From af9995265eb00be2a91ab2c77dcfad189d1c6eb7 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 13:54:26 +0200 Subject: [PATCH] Create a worker to clean clickhouse deleted sites data The plan is to run this weekly, but going to trigger it manually the first few times on cloud --- lib/workers/clickhouse_clean_sites.ex | 63 ++++++++++++++++++ test/workers/clickhouse_clean_sites_test.exs | 69 ++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 lib/workers/clickhouse_clean_sites.ex create mode 100644 test/workers/clickhouse_clean_sites_test.exs diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex new file mode 100644 index 000000000..62541c91f --- /dev/null +++ b/lib/workers/clickhouse_clean_sites.ex @@ -0,0 +1,63 @@ +defmodule Plausible.Workers.ClickhouseCleanSites do + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.IngestRepo + use Oban.Worker, queue: :clickhouse_clean_sites + + import Ecto.Query + + require Logger + + @tables_to_clear [ + "events_v2", + "sessions_v2", + "ingest_counters", + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_pages", + "imported_sources", + "imported_visitors" + ] + + def perform(_job) do + deleted_sites = get_deleted_sites_with_clickhouse_data() + + if length(deleted_sites) > 0 do + Logger.info( + "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" + ) + + site_ids_expr = deleted_sites |> Enum.map(&to_string/1) |> Enum.join(", ") + + for table <- @tables_to_clear do + IngestRepo.query( + "ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})", + [], + if(Mix.env() == :test, do: [mutations_sync: 1], else: []) + ) + end + end + + :ok + end + + def get_deleted_sites_with_clickhouse_data() do + pg_sites = + from(s in Plausible.Site, select: %{id: s.id}) + |> Plausible.Repo.all() + |> Enum.map(&(&1 |> Map.fetch!(:id))) + |> MapSet.new() + + ch_sites = + from(e in "events_v2", group_by: e.site_id, select: %{site_id: e.site_id}) + |> Plausible.ClickhouseRepo.all(timeout: :infinity) + |> Enum.map(& &1.site_id) + |> MapSet.new() + + MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list() + end +end diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs new file mode 100644 index 000000000..debb4c1e6 --- /dev/null +++ b/test/workers/clickhouse_clean_sites_test.exs @@ -0,0 +1,69 @@ +defmodule Plausible.Workers.ClickhouseCleanSitesTest do + use Plausible.DataCase + use Plausible.TestUtils + use Plausible + import Plausible.Factory + + alias Plausible.Workers.ClickhouseCleanSites + + test "deletes data from events and sessions tables" do + site = insert(:site) + deleted_site = insert(:site) + + populate_stats(site, [ + build(:pageview) + ]) + + populate_stats(deleted_site, [ + build(:pageview), + build(:pageview), + build(:imported_visitors), + build(:imported_sources), + build(:imported_pages), + build(:imported_entry_pages), + build(:imported_exit_pages), + build(:imported_locations), + build(:imported_devices), + build(:imported_browsers), + build(:imported_operating_systems) + ]) + + Repo.delete!(deleted_site) + + assert Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + site.id + ) + + ClickhouseCleanSites.perform(nil) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + + assert get_count("events_v2") == 1 + assert get_count("sessions_v2") == 1 + assert get_count("imported_visitors") == 0 + assert get_count("imported_sources") == 0 + assert get_count("imported_pages") == 0 + assert get_count("imported_entry_pages") == 0 + assert get_count("imported_exit_pages") == 0 + assert get_count("imported_locations") == 0 + assert get_count("imported_devices") == 0 + assert get_count("imported_browsers") == 0 + assert get_count("imported_operating_systems") == 0 + end + + def get_count(table) do + %{count: count} = + from(e in table, select: %{count: fragment("count()")}) |> Plausible.ClickhouseRepo.one() + + count + end +end