Create a worker to clean clickhouse deleted sites data
The plan is to run this weekly, but going to trigger it manually the first few times on cloudclean-site-ch-data
parent
edf70d14b6
commit
af9995265e
@ -0,0 +1,63 @@
|
||||
defmodule Plausible.Workers.ClickhouseCleanSites do
|
||||
use Plausible.Repo
|
||||
use Plausible.ClickhouseRepo
|
||||
use Plausible.IngestRepo
|
||||
use Oban.Worker, queue: :clickhouse_clean_sites
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
require Logger
|
||||
|
||||
@tables_to_clear [
|
||||
"events_v2",
|
||||
"sessions_v2",
|
||||
"ingest_counters",
|
||||
"imported_browsers",
|
||||
"imported_devices",
|
||||
"imported_entry_pages",
|
||||
"imported_exit_pages",
|
||||
"imported_locations",
|
||||
"imported_operating_systems",
|
||||
"imported_pages",
|
||||
"imported_sources",
|
||||
"imported_visitors"
|
||||
]
|
||||
|
||||
def perform(_job) do
|
||||
deleted_sites = get_deleted_sites_with_clickhouse_data()
|
||||
|
||||
if length(deleted_sites) > 0 do
|
||||
Logger.info(
|
||||
"Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}"
|
||||
)
|
||||
|
||||
site_ids_expr = deleted_sites |> Enum.map(&to_string/1) |> Enum.join(", ")
|
||||
|
||||
for table <- @tables_to_clear do
|
||||
IngestRepo.query(
|
||||
"ALTER TABLE #{table} DELETE WHERE site_id IN (#{site_ids_expr})",
|
||||
[],
|
||||
if(Mix.env() == :test, do: [mutations_sync: 1], else: [])
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
def get_deleted_sites_with_clickhouse_data() do
|
||||
pg_sites =
|
||||
from(s in Plausible.Site, select: %{id: s.id})
|
||||
|> Plausible.Repo.all()
|
||||
|> Enum.map(&(&1 |> Map.fetch!(:id)))
|
||||
|> MapSet.new()
|
||||
|
||||
ch_sites =
|
||||
from(e in "events_v2", group_by: e.site_id, select: %{site_id: e.site_id})
|
||||
|> Plausible.ClickhouseRepo.all(timeout: :infinity)
|
||||
|> Enum.map(& &1.site_id)
|
||||
|> MapSet.new()
|
||||
|
||||
MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,69 @@
|
||||
defmodule Plausible.Workers.ClickhouseCleanSitesTest do
|
||||
use Plausible.DataCase
|
||||
use Plausible.TestUtils
|
||||
use Plausible
|
||||
import Plausible.Factory
|
||||
|
||||
alias Plausible.Workers.ClickhouseCleanSites
|
||||
|
||||
test "deletes data from events and sessions tables" do
|
||||
site = insert(:site)
|
||||
deleted_site = insert(:site)
|
||||
|
||||
populate_stats(site, [
|
||||
build(:pageview)
|
||||
])
|
||||
|
||||
populate_stats(deleted_site, [
|
||||
build(:pageview),
|
||||
build(:pageview),
|
||||
build(:imported_visitors),
|
||||
build(:imported_sources),
|
||||
build(:imported_pages),
|
||||
build(:imported_entry_pages),
|
||||
build(:imported_exit_pages),
|
||||
build(:imported_locations),
|
||||
build(:imported_devices),
|
||||
build(:imported_browsers),
|
||||
build(:imported_operating_systems)
|
||||
])
|
||||
|
||||
Repo.delete!(deleted_site)
|
||||
|
||||
assert Enum.member?(
|
||||
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
|
||||
deleted_site.id
|
||||
)
|
||||
|
||||
assert not Enum.member?(
|
||||
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
|
||||
site.id
|
||||
)
|
||||
|
||||
ClickhouseCleanSites.perform(nil)
|
||||
|
||||
assert not Enum.member?(
|
||||
ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
|
||||
deleted_site.id
|
||||
)
|
||||
|
||||
assert get_count("events_v2") == 1
|
||||
assert get_count("sessions_v2") == 1
|
||||
assert get_count("imported_visitors") == 0
|
||||
assert get_count("imported_sources") == 0
|
||||
assert get_count("imported_pages") == 0
|
||||
assert get_count("imported_entry_pages") == 0
|
||||
assert get_count("imported_exit_pages") == 0
|
||||
assert get_count("imported_locations") == 0
|
||||
assert get_count("imported_devices") == 0
|
||||
assert get_count("imported_browsers") == 0
|
||||
assert get_count("imported_operating_systems") == 0
|
||||
end
|
||||
|
||||
def get_count(table) do
|
||||
%{count: count} =
|
||||
from(e in table, select: %{count: fragment("count()")}) |> Plausible.ClickhouseRepo.one()
|
||||
|
||||
count
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue