diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex new file mode 100644 index 000000000..e29e2493f --- /dev/null +++ b/lib/workers/clickhouse_clean_sites.ex @@ -0,0 +1,70 @@ +defmodule Plausible.Workers.ClickhouseCleanSites do + @moduledoc """ + Cleans deleted site data from ClickHouse asynchronously. + + We batch up data deletions from ClickHouse as deleting a single site is + just as expensive as deleting many. + """ + + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.IngestRepo + use Oban.Worker, queue: :clickhouse_clean_sites + + import Ecto.Query + + require Logger + + @tables_to_clear [ + "events_v2", + "sessions_v2", + "ingest_counters", + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_pages", + "imported_sources", + "imported_visitors" + ] + + @settings if Mix.env() in [:test, :small_test], do: [mutations_sync: 2], else: [] + + def perform(_job) do + deleted_sites = get_deleted_sites_with_clickhouse_data() + + if length(deleted_sites) > 0 do + Logger.info( + "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" + ) + + site_ids_expr = deleted_sites |> Enum.map_join(", ", &to_string/1) + + for table <- @tables_to_clear do + IngestRepo.query!( + "ALTER TABLE #{table} DELETE WHERE site_id IN #{site_ids_expr}", + [], + settings: @settings + ) + end + end + + :ok + end + + def get_deleted_sites_with_clickhouse_data() do + pg_sites = + from(s in Plausible.Site, select: s.id) + |> Plausible.Repo.all() + |> MapSet.new() + + ch_sites = + from(e in "events_v2", group_by: e.site_id, select: e.site_id) + |> Plausible.ClickhouseRepo.all(timeout: :infinity) + |> MapSet.new() + + MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list() + end +end diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs new file mode 100644 index 000000000..8d85dee2c --- /dev/null +++ b/test/workers/clickhouse_clean_sites_test.exs @@ -0,0 +1,69 @@ +defmodule Plausible.Workers.ClickhouseCleanSitesTest do + use Plausible.DataCase + use Plausible.TestUtils + use Plausible + import Plausible.Factory + + alias Plausible.Workers.ClickhouseCleanSites + + test "deletes data from events and sessions tables" do + site = insert(:site) + deleted_site = insert(:site) + + populate_stats(site, [ + build(:pageview) + ]) + + populate_stats(deleted_site, [ + build(:pageview), + build(:pageview), + build(:imported_visitors), + build(:imported_sources), + build(:imported_pages), + build(:imported_entry_pages), + build(:imported_exit_pages), + build(:imported_locations), + build(:imported_devices), + build(:imported_browsers), + build(:imported_operating_systems) + ]) + + Repo.delete!(deleted_site) + + assert Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + site.id + ) + + ClickhouseCleanSites.perform(nil) + + assert_count(deleted_site, "events_v2", 0) + assert_count(deleted_site, "sessions_v2", 0) + assert_count(deleted_site, "imported_visitors", 0) + assert_count(deleted_site, "imported_sources", 0) + assert_count(deleted_site, "imported_pages", 0) + assert_count(deleted_site, "imported_entry_pages", 0) + assert_count(deleted_site, "imported_exit_pages", 0) + assert_count(deleted_site, "imported_locations", 0) + assert_count(deleted_site, "imported_devices", 0) + assert_count(deleted_site, "imported_browsers", 0) + assert_count(deleted_site, "imported_operating_systems", 0) + assert_count(site, "events_v2", 1) + assert_count(site, "sessions_v2", 1) + + assert not Enum.member?( + ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(), + deleted_site.id + ) + end + + def assert_count(site, table, expected_count) do + q = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id) + await_clickhouse_count(q, expected_count) + end +end