Delay stats deletions (#2632)
* Implement Site removal transaction * Implement Stats removal Oban worker * Configure site removal queue * Call Site.Removal.run() instead of Purge.delete_site! * Test site/stats removal * Remove FIXME - filed a ticket * Over-communicate lenghty deletion process to the userspull/2633/head
parent
1b9e6d9ae5
commit
8f9f032968
@ -0,0 +1,41 @@
|
||||
defmodule Plausible.Site.Removal do
|
||||
@moduledoc """
|
||||
A service responsible for site and its stats deletion.
|
||||
The site deletion alone is done in postgres and is executed first,
|
||||
the latter deletions (events, sessions and imported tables in clickhouse)
|
||||
are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`.
|
||||
|
||||
This is to avoid race condition in which the site is deleted, but stats
|
||||
writes are pending (either in the buffers or are about to be buffered, due
|
||||
to Sites.Cache keeping the now obsolete record until refresh is triggered).
|
||||
"""
|
||||
@stats_deletion_delay_seconds 60 * 20
|
||||
|
||||
alias Plausible.Workers.StatsRemoval
|
||||
alias Plausible.Repo
|
||||
alias Ecto.Multi
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
@spec stats_deletion_delay_seconds() :: pos_integer()
|
||||
def stats_deletion_delay_seconds() do
|
||||
@stats_deletion_delay_seconds
|
||||
end
|
||||
|
||||
@spec run(String.t()) :: {:ok, map()}
|
||||
def run(domain) do
|
||||
site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain
|
||||
|
||||
Multi.new()
|
||||
|> Multi.run(:site_id, fn _, _ ->
|
||||
{:ok, Repo.one(from s in site_by_domain_q, select: s.id)}
|
||||
end)
|
||||
|> Multi.delete_all(:delete_all, site_by_domain_q)
|
||||
|> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} ->
|
||||
StatsRemoval.new(%{domain: domain, site_id: site_id},
|
||||
schedule_in: stats_deletion_delay_seconds()
|
||||
)
|
||||
end)
|
||||
|> Repo.transaction()
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,59 @@
|
||||
defmodule Plausible.Workers.StatsRemoval do
|
||||
@moduledoc """
|
||||
Asynchronous worker firing deletion mutations to clickhouse.
|
||||
For now only ALTER TABLE deletions are supported. Experimental
|
||||
DELETE FROM support is going to be introduced once production db
|
||||
is upgraded.
|
||||
|
||||
At most 3 attempts are made, with 15m backoff value.
|
||||
|
||||
Imported stats tables keep site reference through a numeric id, whilist
|
||||
events and sessions store domain as-is - hence two different deletes,
|
||||
one of which cannot be performed anymore once the site identifier is permanently
|
||||
gone from postgres.
|
||||
"""
|
||||
use Plausible.Repo
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :site_stats_removal,
|
||||
max_attempts: 3,
|
||||
unique: [period: :infinity, fields: [:args]]
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%{args: args}) do
|
||||
domain = Map.fetch!(args, "domain")
|
||||
site_id = Map.get(args, "site_id")
|
||||
|
||||
imported_result = delete_imported!(site_id)
|
||||
native_result = delete_native!(domain)
|
||||
|
||||
{:ok, Map.merge(imported_result, native_result)}
|
||||
end
|
||||
|
||||
@impl Oban.Worker
|
||||
def backoff(_job) do
|
||||
15 * 60
|
||||
end
|
||||
|
||||
defp delete_imported!(nil) do
|
||||
%{}
|
||||
end
|
||||
|
||||
defp delete_imported!(id) when is_integer(id) do
|
||||
Enum.map(Plausible.Imported.tables(), fn table ->
|
||||
sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"
|
||||
{table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])}
|
||||
end)
|
||||
|> Enum.into(%{})
|
||||
end
|
||||
|
||||
defp delete_native!(domain) do
|
||||
events_sql = "ALTER TABLE events DELETE WHERE domain = ?"
|
||||
sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"
|
||||
|
||||
%{
|
||||
"events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]),
|
||||
"sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain])
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,116 @@
|
||||
defmodule Plausible.Site.SiteRemovalTest do
|
||||
use Plausible.DataCase, async: true
|
||||
use Oban.Testing, repo: Plausible.Repo
|
||||
|
||||
alias Plausible.Site.Removal
|
||||
alias Plausible.Sites
|
||||
alias Plausible.Workers.StatsRemoval
|
||||
|
||||
describe "execution and scheduling" do
|
||||
test "site from postgres is immediately deleted" do
|
||||
site = insert(:site)
|
||||
assert {:ok, context} = Removal.run(site.domain)
|
||||
assert context.delete_all == {1, nil}
|
||||
assert context.site_id == site.id
|
||||
refute Sites.get_by_domain(site.domain)
|
||||
end
|
||||
|
||||
test "deletion is idempotent" do
|
||||
assert {:ok, context} = Removal.run("some.example.com")
|
||||
assert context.delete_all == {0, nil}
|
||||
end
|
||||
|
||||
test "stats deletion job is scheduled when no site exists in postgres" do
|
||||
assert {:ok, _} = Removal.run("a.domain.example.com")
|
||||
|
||||
assert_enqueued(
|
||||
worker: StatsRemoval,
|
||||
args: %{"domain" => "a.domain.example.com", "site_id" => nil}
|
||||
)
|
||||
end
|
||||
|
||||
test "stats deletion job is scheduled when site exists in postgres" do
|
||||
site = insert(:site)
|
||||
assert {:ok, _} = Removal.run(site.domain)
|
||||
|
||||
assert_enqueued(
|
||||
worker: StatsRemoval,
|
||||
args: %{"domain" => site.domain, "site_id" => site.id}
|
||||
)
|
||||
end
|
||||
|
||||
test "stats deletion is always scheduled ~20m in the future" do
|
||||
assert {:ok, _} = Removal.run("foo.example.com")
|
||||
|
||||
in_20m = DateTime.add(DateTime.utc_now(), 1200, :second)
|
||||
|
||||
assert_enqueued(
|
||||
worker: StatsRemoval,
|
||||
scheduled_at: {in_20m, delta: 5}
|
||||
)
|
||||
end
|
||||
|
||||
test "stats deletion is always scheduled late enough for sites cache to expire" do
|
||||
delay_ms = Removal.stats_deletion_delay_seconds() * 1000
|
||||
assert delay_ms > Plausible.Site.Cache.Warmer.interval()
|
||||
end
|
||||
end
|
||||
|
||||
describe "the background worker" do
|
||||
test "the job runs deletes at clickhouse" do
|
||||
assert {:ok, %{"events" => r, "sessions" => r}} =
|
||||
perform_job(StatsRemoval, %{"domain" => "foo.example.com"})
|
||||
|
||||
assert %Clickhousex.Result{command: :updated} = r
|
||||
|
||||
assert {:ok, %{"events" => r, "sessions" => r, "imported_browsers" => r}} =
|
||||
perform_job(StatsRemoval, %{"domain" => "foo.example.com", "site_id" => 777})
|
||||
|
||||
assert %Clickhousex.Result{command: :updated} = r
|
||||
end
|
||||
end
|
||||
|
||||
describe "integration" do
|
||||
setup do
|
||||
site = insert(:site, stats_start_date: ~D[2020-01-01])
|
||||
|
||||
populate_stats(site, [
|
||||
build(:pageview),
|
||||
build(:imported_visitors),
|
||||
build(:imported_sources),
|
||||
build(:imported_pages),
|
||||
build(:imported_entry_pages),
|
||||
build(:imported_exit_pages),
|
||||
build(:imported_locations),
|
||||
build(:imported_devices),
|
||||
build(:imported_browsers),
|
||||
build(:imported_operating_systems)
|
||||
])
|
||||
|
||||
{:ok, %{site: site}}
|
||||
end
|
||||
|
||||
test "the job actually removes stats from clickhouse", %{site: site} do
|
||||
Enum.each(Plausible.Imported.tables(), fn table ->
|
||||
query = from(imported in table, where: imported.site_id == ^site.id)
|
||||
assert await_clickhouse_count(query, 1)
|
||||
end)
|
||||
|
||||
events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)
|
||||
assert await_clickhouse_count(events_query, 1)
|
||||
|
||||
sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)
|
||||
assert await_clickhouse_count(sessions_query, 1)
|
||||
|
||||
perform_job(StatsRemoval, %{"domain" => site.domain, "site_id" => site.id})
|
||||
|
||||
assert await_clickhouse_count(events_query, 0)
|
||||
assert await_clickhouse_count(sessions_query, 0)
|
||||
|
||||
Enum.each(Plausible.Imported.tables(), fn table ->
|
||||
query = from(imported in table, where: imported.site_id == ^site.id)
|
||||
assert await_clickhouse_count(query, 0)
|
||||
end)
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue