Create a worker to clean clickhouse deleted sites data

The plan is to run this weekly, but going to trigger it manually the first few times on cloud
clean-site-ch-data
Karl-Aksel Puulmann 3 years ago
parent edf70d14b6
commit af9995265e

@ -0,0 +1,63 @@
defmodule Plausible.Workers.ClickhouseCleanSites do
use Plausible.Repo
use Plausible.ClickhouseRepo
use Plausible.IngestRepo
use Oban.Worker, queue: :clickhouse_clean_sites
import Ecto.Query
require Logger
@tables_to_clear [
"events_v2",
"sessions_v2",
"ingest_counters",
"imported_browsers",
"imported_devices",
"imported_entry_pages",
"imported_exit_pages",
"imported_locations",
"imported_operating_systems",
"imported_pages",
"imported_sources",
"imported_visitors"
]
def perform(_job) do
deleted_sites = get_deleted_sites_with_clickhouse_data()
if length(deleted_sites) > 0 do
Logger.info(
"Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}"
)
site_ids_expr = deleted_sites |> Enum.map(&to_string/1) |> Enum.join(", ")
for table <- @tables_to_clear do
IngestRepo.query(
"ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})",
[],
if(Mix.env() == :test, do: [mutations_sync: 1], else: [])
)
end
end
:ok
end
def get_deleted_sites_with_clickhouse_data() do
pg_sites =
from(s in Plausible.Site, select: %{id: s.id})
|> Plausible.Repo.all()
|> Enum.map(&(&1 |> Map.fetch!(:id)))
|> MapSet.new()
ch_sites =
from(e in "events_v2", group_by: e.site_id, select: %{site_id: e.site_id})
|> Plausible.ClickhouseRepo.all(timeout: :infinity)
|> Enum.map(& &1.site_id)
|> MapSet.new()
MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()
end
end

@ -0,0 +1,69 @@
defmodule Plausible.Workers.ClickhouseCleanSitesTest do
use Plausible.DataCase
use Plausible.TestUtils
use Plausible
import Plausible.Factory
alias Plausible.Workers.ClickhouseCleanSites
test "deletes data from events and sessions tables" do
site = insert(:site)
deleted_site = insert(:site)
populate_stats(site, [
build(:pageview)
])
populate_stats(deleted_site, [
build(:pageview),
build(:pageview),
build(:imported_visitors),
build(:imported_sources),
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
build(:imported_operating_systems)
])
Repo.delete!(deleted_site)
assert Enum.member?(
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
deleted_site.id
)
assert not Enum.member?(
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
site.id
)
ClickhouseCleanSites.perform(nil)
assert not Enum.member?(
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
deleted_site.id
)
assert get_count("events_v2") == 1
assert get_count("sessions_v2") == 1
assert get_count("imported_visitors") == 0
assert get_count("imported_sources") == 0
assert get_count("imported_pages") == 0
assert get_count("imported_entry_pages") == 0
assert get_count("imported_exit_pages") == 0
assert get_count("imported_locations") == 0
assert get_count("imported_devices") == 0
assert get_count("imported_browsers") == 0
assert get_count("imported_operating_systems") == 0
end
def get_count(table) do
%{count: count} =
from(e in table, select: %{count: fragment("count()")}) |> Plausible.ClickhouseRepo.one()
count
end
end
Loading…
Cancel
Save