Delay stats deletions (#2632)

* Implement Site removal transaction

* Implement Stats removal Oban worker

* Configure site removal queue

* Call Site.Removal.run() instead of Purge.delete_site!

* Test site/stats removal

* Remove FIXME - filed a ticket

* Over-communicate lenghty deletion process to the users
pull/2633/head
Adam Rutkowski 4 years ago committed by GitHub
parent 1b9e6d9ae5
commit 8f9f032968
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -352,7 +352,8 @@ base_queues = [
site_setup_emails: 1,
clean_email_verification_codes: 1,
clean_invitations: 1,
google_analytics_imports: 1
google_analytics_imports: 1,
site_stats_removal: 1
]
cloud_queues = [

@ -9,18 +9,6 @@ defmodule Plausible.Purge do
- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)
"""
@doc """
Deletes a site and all associated stats.
"""
@spec delete_site!(Plausible.Site.t()) :: :ok
def delete_site!(site) do
delete_native_stats!(site)
delete_imported_stats!(site)
Plausible.Repo.delete!(site)
:ok
end
@spec delete_imported_stats!(Plausible.Site.t()) :: :ok
@doc """
Deletes imported stats from Google Analytics, and clears the

@ -0,0 +1,41 @@
defmodule Plausible.Site.Removal do
@moduledoc """
A service responsible for site and its stats deletion.
The site deletion alone is done in postgres and is executed first,
the latter deletions (events, sessions and imported tables in clickhouse)
are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`.
This is to avoid race condition in which the site is deleted, but stats
writes are pending (either in the buffers or are about to be buffered, due
to Sites.Cache keeping the now obsolete record until refresh is triggered).
"""
@stats_deletion_delay_seconds 60 * 20
alias Plausible.Workers.StatsRemoval
alias Plausible.Repo
alias Ecto.Multi
import Ecto.Query
@spec stats_deletion_delay_seconds() :: pos_integer()
def stats_deletion_delay_seconds() do
@stats_deletion_delay_seconds
end
@spec run(String.t()) :: {:ok, map()}
def run(domain) do
site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain
Multi.new()
|> Multi.run(:site_id, fn _, _ ->
{:ok, Repo.one(from s in site_by_domain_q, select: s.id)}
end)
|> Multi.delete_all(:delete_all, site_by_domain_q)
|> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} ->
StatsRemoval.new(%{domain: domain, site_id: site_id},
schedule_in: stats_deletion_delay_seconds()
)
end)
|> Repo.transaction()
end
end

@ -42,7 +42,7 @@ defmodule PlausibleWeb.Api.ExternalSitesController do
site = Sites.get_for_user(conn.assigns[:current_user].id, site_id, [:owner])
if site do
Plausible.Purge.delete_site!(site)
{:ok, _} = Plausible.Site.Removal.run(site.domain)
json(conn, %{"deleted" => true})
else
H.not_found(conn, "Site could not be found")

@ -542,7 +542,7 @@ defmodule PlausibleWeb.AuthController do
Repo.delete!(membership)
if membership.role == :owner do
Plausible.Purge.delete_site!(membership.site)
Plausible.Site.Removal.run(membership.site.domain)
end
end

@ -359,10 +359,10 @@ defmodule PlausibleWeb.SiteController do
def delete_site(conn, _params) do
site = conn.assigns[:site]
Plausible.Purge.delete_site!(site)
Plausible.Site.Removal.run(site.domain)
conn
|> put_flash(:success, "Site deleted successfully along with all pageviews")
|> put_flash(:success, "Your site and page views deletion process has started.")
|> redirect(to: "/sites")
end

@ -38,7 +38,7 @@
Delete site
</p>
<p class="text-sm leading-5 text-gray-500 dark:text-gray-200">
Permanently remove all stats and the site configuration too. This process takes a few hours so you won't be able to immediately register the same site again
Permanently remove all stats and the site configuration too. This process may take up to 48 hours so you won't be able to immediately register the same site again
</p>
</div>
<%= link "Delete #{@site.domain}", to: "/#{URI.encode_www_form(@site.domain)}", method: :delete, class: "inline-block px-4 py-2 border border-transparent font-medium rounded-md text-red-700 dark:text-red-800 bg-red-100 dark:bg-red-200 hover:bg-red-50 dark:hover:bg-red-300 focus:outline-none focus:border-red-300 focus:ring active:bg-red-200 transition ease-in-out duration-150 sm:text-sm sm:leading-5", data: [confirm: "Deleting the site data cannot be reversed. Are you sure?"] %>

@ -0,0 +1,59 @@
defmodule Plausible.Workers.StatsRemoval do
@moduledoc """
Asynchronous worker firing deletion mutations to clickhouse.
For now only ALTER TABLE deletions are supported. Experimental
DELETE FROM support is going to be introduced once production db
is upgraded.
At most 3 attempts are made, with 15m backoff value.
Imported stats tables keep site reference through a numeric id, whilist
events and sessions store domain as-is - hence two different deletes,
one of which cannot be performed anymore once the site identifier is permanently
gone from postgres.
"""
use Plausible.Repo
use Oban.Worker,
queue: :site_stats_removal,
max_attempts: 3,
unique: [period: :infinity, fields: [:args]]
@impl Oban.Worker
def perform(%{args: args}) do
domain = Map.fetch!(args, "domain")
site_id = Map.get(args, "site_id")
imported_result = delete_imported!(site_id)
native_result = delete_native!(domain)
{:ok, Map.merge(imported_result, native_result)}
end
@impl Oban.Worker
def backoff(_job) do
15 * 60
end
defp delete_imported!(nil) do
%{}
end
defp delete_imported!(id) when is_integer(id) do
Enum.map(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"
{table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])}
end)
|> Enum.into(%{})
end
defp delete_native!(domain) do
events_sql = "ALTER TABLE events DELETE WHERE domain = ?"
sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"
%{
"events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]),
"sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain])
}
end
end

@ -20,28 +20,17 @@ defmodule Plausible.PurgeTest do
{:ok, %{site: site}}
end
defp assert_count(query, expected) do
assert eventually(
fn ->
count = Plausible.ClickhouseRepo.aggregate(query, :count)
{count == expected, count}
end,
200,
10
)
end
test "delete_imported_stats!/1 deletes imported data", %{site: site} do
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert_count(query, 1)
assert await_clickhouse_count(query, 1)
end)
assert :ok == Plausible.Purge.delete_imported_stats!(site)
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert_count(query, 0)
assert await_clickhouse_count(query, 0)
end)
end
@ -52,38 +41,19 @@ defmodule Plausible.PurgeTest do
test "delete_native_stats!/1 deletes native stats", %{site: site} do
events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)
assert_count(events_query, 1)
assert await_clickhouse_count(events_query, 1)
sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)
assert_count(sessions_query, 1)
assert await_clickhouse_count(sessions_query, 1)
assert :ok == Plausible.Purge.delete_native_stats!(site)
assert_count(events_query, 0)
assert_count(sessions_query, 0)
assert await_clickhouse_count(events_query, 0)
assert await_clickhouse_count(sessions_query, 0)
end
test "delete_native_stats!/1 resets stats_start_date", %{site: site} do
assert :ok == Plausible.Purge.delete_native_stats!(site)
assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)
end
test "delete_site!/1 deletes the site and all stats", %{site: site} do
events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)
assert_count(events_query, 1)
sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)
assert_count(sessions_query, 1)
assert :ok == Plausible.Purge.delete_site!(site)
assert nil == Plausible.Repo.reload(site)
assert_count(events_query, 0)
assert_count(sessions_query, 0)
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert_count(query, 0)
end)
end
end

@ -0,0 +1,116 @@
defmodule Plausible.Site.SiteRemovalTest do
use Plausible.DataCase, async: true
use Oban.Testing, repo: Plausible.Repo
alias Plausible.Site.Removal
alias Plausible.Sites
alias Plausible.Workers.StatsRemoval
describe "execution and scheduling" do
test "site from postgres is immediately deleted" do
site = insert(:site)
assert {:ok, context} = Removal.run(site.domain)
assert context.delete_all == {1, nil}
assert context.site_id == site.id
refute Sites.get_by_domain(site.domain)
end
test "deletion is idempotent" do
assert {:ok, context} = Removal.run("some.example.com")
assert context.delete_all == {0, nil}
end
test "stats deletion job is scheduled when no site exists in postgres" do
assert {:ok, _} = Removal.run("a.domain.example.com")
assert_enqueued(
worker: StatsRemoval,
args: %{"domain" => "a.domain.example.com", "site_id" => nil}
)
end
test "stats deletion job is scheduled when site exists in postgres" do
site = insert(:site)
assert {:ok, _} = Removal.run(site.domain)
assert_enqueued(
worker: StatsRemoval,
args: %{"domain" => site.domain, "site_id" => site.id}
)
end
test "stats deletion is always scheduled ~20m in the future" do
assert {:ok, _} = Removal.run("foo.example.com")
in_20m = DateTime.add(DateTime.utc_now(), 1200, :second)
assert_enqueued(
worker: StatsRemoval,
scheduled_at: {in_20m, delta: 5}
)
end
test "stats deletion is always scheduled late enough for sites cache to expire" do
delay_ms = Removal.stats_deletion_delay_seconds() * 1000
assert delay_ms > Plausible.Site.Cache.Warmer.interval()
end
end
describe "the background worker" do
test "the job runs deletes at clickhouse" do
assert {:ok, %{"events" => r, "sessions" => r}} =
perform_job(StatsRemoval, %{"domain" => "foo.example.com"})
assert %Clickhousex.Result{command: :updated} = r
assert {:ok, %{"events" => r, "sessions" => r, "imported_browsers" => r}} =
perform_job(StatsRemoval, %{"domain" => "foo.example.com", "site_id" => 777})
assert %Clickhousex.Result{command: :updated} = r
end
end
describe "integration" do
setup do
site = insert(:site, stats_start_date: ~D[2020-01-01])
populate_stats(site, [
build(:pageview),
build(:imported_visitors),
build(:imported_sources),
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
build(:imported_operating_systems)
])
{:ok, %{site: site}}
end
test "the job actually removes stats from clickhouse", %{site: site} do
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert await_clickhouse_count(query, 1)
end)
events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)
assert await_clickhouse_count(events_query, 1)
sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)
assert await_clickhouse_count(sessions_query, 1)
perform_job(StatsRemoval, %{"domain" => site.domain, "site_id" => site.id})
assert await_clickhouse_count(events_query, 0)
assert await_clickhouse_count(sessions_query, 0)
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert await_clickhouse_count(query, 0)
end)
end
end
end

@ -184,4 +184,16 @@ defmodule Plausible.TestUtils do
end
end)
end
def await_clickhouse_count(query, expected) do
eventually(
fn ->
count = Plausible.ClickhouseRepo.aggregate(query, :count)
{count == expected, count}
end,
200,
10
)
end
end

Loading…
Cancel
Save