From 8f9f03296818caadefb95b709d91699395fbe5df Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Tue, 31 Jan 2023 20:11:04 +0100 Subject: [PATCH] Delay stats deletions (#2632) * Implement Site removal transaction * Implement Stats removal Oban worker * Configure site removal queue * Call Site.Removal.run() instead of Purge.delete_site! * Test site/stats removal * Remove FIXME - filed a ticket * Over-communicate lenghty deletion process to the users --- config/runtime.exs | 3 +- lib/plausible/purge.ex | 12 -- lib/plausible/site/removal.ex | 41 +++++++ .../api/external_sites_controller.ex | 2 +- .../controllers/auth_controller.ex | 2 +- .../controllers/site_controller.ex | 4 +- .../site/settings_danger_zone.html.eex | 2 +- lib/workers/stats_removal.ex | 59 +++++++++ test/plausible/purge_test.exs | 42 +------ test/plausible/site/site_removal_test.exs | 116 ++++++++++++++++++ test/support/test_utils.ex | 12 ++ 11 files changed, 241 insertions(+), 54 deletions(-) create mode 100644 lib/plausible/site/removal.ex create mode 100644 lib/workers/stats_removal.ex create mode 100644 test/plausible/site/site_removal_test.exs diff --git a/config/runtime.exs b/config/runtime.exs index 53ed3ca92..f3337b46e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -352,7 +352,8 @@ base_queues = [ site_setup_emails: 1, clean_email_verification_codes: 1, clean_invitations: 1, - google_analytics_imports: 1 + google_analytics_imports: 1, + site_stats_removal: 1 ] cloud_queues = [ diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex index 3bb91c9a0..13865a700 100644 --- a/lib/plausible/purge.ex +++ b/lib/plausible/purge.ex @@ -9,18 +9,6 @@ defmodule Plausible.Purge do - [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries) """ - @doc """ - Deletes a site and all associated stats. - """ - @spec delete_site!(Plausible.Site.t()) :: :ok - def delete_site!(site) do - delete_native_stats!(site) - delete_imported_stats!(site) - Plausible.Repo.delete!(site) - - :ok - end - @spec delete_imported_stats!(Plausible.Site.t()) :: :ok @doc """ Deletes imported stats from Google Analytics, and clears the diff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.ex new file mode 100644 index 000000000..c8bd91ad7 --- /dev/null +++ b/lib/plausible/site/removal.ex @@ -0,0 +1,41 @@ +defmodule Plausible.Site.Removal do + @moduledoc """ + A service responsible for site and its stats deletion. + The site deletion alone is done in postgres and is executed first, + the latter deletions (events, sessions and imported tables in clickhouse) + are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`. + + This is to avoid race condition in which the site is deleted, but stats + writes are pending (either in the buffers or are about to be buffered, due + to Sites.Cache keeping the now obsolete record until refresh is triggered). + """ + @stats_deletion_delay_seconds 60 * 20 + + alias Plausible.Workers.StatsRemoval + alias Plausible.Repo + alias Ecto.Multi + + import Ecto.Query + + @spec stats_deletion_delay_seconds() :: pos_integer() + def stats_deletion_delay_seconds() do + @stats_deletion_delay_seconds + end + + @spec run(String.t()) :: {:ok, map()} + def run(domain) do + site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain + + Multi.new() + |> Multi.run(:site_id, fn _, _ -> + {:ok, Repo.one(from s in site_by_domain_q, select: s.id)} + end) + |> Multi.delete_all(:delete_all, site_by_domain_q) + |> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} -> + StatsRemoval.new(%{domain: domain, site_id: site_id}, + schedule_in: stats_deletion_delay_seconds() + ) + end) + |> Repo.transaction() + end +end diff --git a/lib/plausible_web/controllers/api/external_sites_controller.ex b/lib/plausible_web/controllers/api/external_sites_controller.ex index a754d98fa..57e18e0b9 100644 --- a/lib/plausible_web/controllers/api/external_sites_controller.ex +++ b/lib/plausible_web/controllers/api/external_sites_controller.ex @@ -42,7 +42,7 @@ defmodule PlausibleWeb.Api.ExternalSitesController do site = Sites.get_for_user(conn.assigns[:current_user].id, site_id, [:owner]) if site do - Plausible.Purge.delete_site!(site) + {:ok, _} = Plausible.Site.Removal.run(site.domain) json(conn, %{"deleted" => true}) else H.not_found(conn, "Site could not be found") diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 51721bbe1..3ae2de68f 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -542,7 +542,7 @@ defmodule PlausibleWeb.AuthController do Repo.delete!(membership) if membership.role == :owner do - Plausible.Purge.delete_site!(membership.site) + Plausible.Site.Removal.run(membership.site.domain) end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index b43b7c8c6..145b6fc3c 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -359,10 +359,10 @@ defmodule PlausibleWeb.SiteController do def delete_site(conn, _params) do site = conn.assigns[:site] - Plausible.Purge.delete_site!(site) + Plausible.Site.Removal.run(site.domain) conn - |> put_flash(:success, "Site deleted successfully along with all pageviews") + |> put_flash(:success, "Your site and page views deletion process has started.") |> redirect(to: "/sites") end diff --git a/lib/plausible_web/templates/site/settings_danger_zone.html.eex b/lib/plausible_web/templates/site/settings_danger_zone.html.eex index bf9029513..59d28293d 100644 --- a/lib/plausible_web/templates/site/settings_danger_zone.html.eex +++ b/lib/plausible_web/templates/site/settings_danger_zone.html.eex @@ -38,7 +38,7 @@ Delete site

- Permanently remove all stats and the site configuration too. This process takes a few hours so you won't be able to immediately register the same site again + Permanently remove all stats and the site configuration too. This process may take up to 48 hours so you won't be able to immediately register the same site again

<%= link "Delete #{@site.domain}", to: "/#{URI.encode_www_form(@site.domain)}", method: :delete, class: "inline-block px-4 py-2 border border-transparent font-medium rounded-md text-red-700 dark:text-red-800 bg-red-100 dark:bg-red-200 hover:bg-red-50 dark:hover:bg-red-300 focus:outline-none focus:border-red-300 focus:ring active:bg-red-200 transition ease-in-out duration-150 sm:text-sm sm:leading-5", data: [confirm: "Deleting the site data cannot be reversed. Are you sure?"] %> diff --git a/lib/workers/stats_removal.ex b/lib/workers/stats_removal.ex new file mode 100644 index 000000000..6ca86ba0d --- /dev/null +++ b/lib/workers/stats_removal.ex @@ -0,0 +1,59 @@ +defmodule Plausible.Workers.StatsRemoval do + @moduledoc """ + Asynchronous worker firing deletion mutations to clickhouse. + For now only ALTER TABLE deletions are supported. Experimental + DELETE FROM support is going to be introduced once production db + is upgraded. + + At most 3 attempts are made, with 15m backoff value. + + Imported stats tables keep site reference through a numeric id, whilist + events and sessions store domain as-is - hence two different deletes, + one of which cannot be performed anymore once the site identifier is permanently + gone from postgres. + """ + use Plausible.Repo + + use Oban.Worker, + queue: :site_stats_removal, + max_attempts: 3, + unique: [period: :infinity, fields: [:args]] + + @impl Oban.Worker + def perform(%{args: args}) do + domain = Map.fetch!(args, "domain") + site_id = Map.get(args, "site_id") + + imported_result = delete_imported!(site_id) + native_result = delete_native!(domain) + + {:ok, Map.merge(imported_result, native_result)} + end + + @impl Oban.Worker + def backoff(_job) do + 15 * 60 + end + + defp delete_imported!(nil) do + %{} + end + + defp delete_imported!(id) when is_integer(id) do + Enum.map(Plausible.Imported.tables(), fn table -> + sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?" + {table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])} + end) + |> Enum.into(%{}) + end + + defp delete_native!(domain) do + events_sql = "ALTER TABLE events DELETE WHERE domain = ?" + sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?" + + %{ + "events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]), + "sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain]) + } + end +end diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exs index 1efc98a83..a05fc51e9 100644 --- a/test/plausible/purge_test.exs +++ b/test/plausible/purge_test.exs @@ -20,28 +20,17 @@ defmodule Plausible.PurgeTest do {:ok, %{site: site}} end - defp assert_count(query, expected) do - assert eventually( - fn -> - count = Plausible.ClickhouseRepo.aggregate(query, :count) - {count == expected, count} - end, - 200, - 10 - ) - end - test "delete_imported_stats!/1 deletes imported data", %{site: site} do Enum.each(Plausible.Imported.tables(), fn table -> query = from(imported in table, where: imported.site_id == ^site.id) - assert_count(query, 1) + assert await_clickhouse_count(query, 1) end) assert :ok == Plausible.Purge.delete_imported_stats!(site) Enum.each(Plausible.Imported.tables(), fn table -> query = from(imported in table, where: imported.site_id == ^site.id) - assert_count(query, 0) + assert await_clickhouse_count(query, 0) end) end @@ -52,38 +41,19 @@ defmodule Plausible.PurgeTest do test "delete_native_stats!/1 deletes native stats", %{site: site} do events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain) - assert_count(events_query, 1) + assert await_clickhouse_count(events_query, 1) sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain) - assert_count(sessions_query, 1) + assert await_clickhouse_count(sessions_query, 1) assert :ok == Plausible.Purge.delete_native_stats!(site) - assert_count(events_query, 0) - assert_count(sessions_query, 0) + assert await_clickhouse_count(events_query, 0) + assert await_clickhouse_count(sessions_query, 0) end test "delete_native_stats!/1 resets stats_start_date", %{site: site} do assert :ok == Plausible.Purge.delete_native_stats!(site) assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site) end - - test "delete_site!/1 deletes the site and all stats", %{site: site} do - events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain) - assert_count(events_query, 1) - - sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain) - assert_count(sessions_query, 1) - - assert :ok == Plausible.Purge.delete_site!(site) - assert nil == Plausible.Repo.reload(site) - - assert_count(events_query, 0) - assert_count(sessions_query, 0) - - Enum.each(Plausible.Imported.tables(), fn table -> - query = from(imported in table, where: imported.site_id == ^site.id) - assert_count(query, 0) - end) - end end diff --git a/test/plausible/site/site_removal_test.exs b/test/plausible/site/site_removal_test.exs new file mode 100644 index 000000000..544acbf17 --- /dev/null +++ b/test/plausible/site/site_removal_test.exs @@ -0,0 +1,116 @@ +defmodule Plausible.Site.SiteRemovalTest do + use Plausible.DataCase, async: true + use Oban.Testing, repo: Plausible.Repo + + alias Plausible.Site.Removal + alias Plausible.Sites + alias Plausible.Workers.StatsRemoval + + describe "execution and scheduling" do + test "site from postgres is immediately deleted" do + site = insert(:site) + assert {:ok, context} = Removal.run(site.domain) + assert context.delete_all == {1, nil} + assert context.site_id == site.id + refute Sites.get_by_domain(site.domain) + end + + test "deletion is idempotent" do + assert {:ok, context} = Removal.run("some.example.com") + assert context.delete_all == {0, nil} + end + + test "stats deletion job is scheduled when no site exists in postgres" do + assert {:ok, _} = Removal.run("a.domain.example.com") + + assert_enqueued( + worker: StatsRemoval, + args: %{"domain" => "a.domain.example.com", "site_id" => nil} + ) + end + + test "stats deletion job is scheduled when site exists in postgres" do + site = insert(:site) + assert {:ok, _} = Removal.run(site.domain) + + assert_enqueued( + worker: StatsRemoval, + args: %{"domain" => site.domain, "site_id" => site.id} + ) + end + + test "stats deletion is always scheduled ~20m in the future" do + assert {:ok, _} = Removal.run("foo.example.com") + + in_20m = DateTime.add(DateTime.utc_now(), 1200, :second) + + assert_enqueued( + worker: StatsRemoval, + scheduled_at: {in_20m, delta: 5} + ) + end + + test "stats deletion is always scheduled late enough for sites cache to expire" do + delay_ms = Removal.stats_deletion_delay_seconds() * 1000 + assert delay_ms > Plausible.Site.Cache.Warmer.interval() + end + end + + describe "the background worker" do + test "the job runs deletes at clickhouse" do + assert {:ok, %{"events" => r, "sessions" => r}} = + perform_job(StatsRemoval, %{"domain" => "foo.example.com"}) + + assert %Clickhousex.Result{command: :updated} = r + + assert {:ok, %{"events" => r, "sessions" => r, "imported_browsers" => r}} = + perform_job(StatsRemoval, %{"domain" => "foo.example.com", "site_id" => 777}) + + assert %Clickhousex.Result{command: :updated} = r + end + end + + describe "integration" do + setup do + site = insert(:site, stats_start_date: ~D[2020-01-01]) + + populate_stats(site, [ + build(:pageview), + build(:imported_visitors), + build(:imported_sources), + build(:imported_pages), + build(:imported_entry_pages), + build(:imported_exit_pages), + build(:imported_locations), + build(:imported_devices), + build(:imported_browsers), + build(:imported_operating_systems) + ]) + + {:ok, %{site: site}} + end + + test "the job actually removes stats from clickhouse", %{site: site} do + Enum.each(Plausible.Imported.tables(), fn table -> + query = from(imported in table, where: imported.site_id == ^site.id) + assert await_clickhouse_count(query, 1) + end) + + events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain) + assert await_clickhouse_count(events_query, 1) + + sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain) + assert await_clickhouse_count(sessions_query, 1) + + perform_job(StatsRemoval, %{"domain" => site.domain, "site_id" => site.id}) + + assert await_clickhouse_count(events_query, 0) + assert await_clickhouse_count(sessions_query, 0) + + Enum.each(Plausible.Imported.tables(), fn table -> + query = from(imported in table, where: imported.site_id == ^site.id) + assert await_clickhouse_count(query, 0) + end) + end + end +end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index ed2b41f07..698c4e11b 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -184,4 +184,16 @@ defmodule Plausible.TestUtils do end end) end + + def await_clickhouse_count(query, expected) do + eventually( + fn -> + count = Plausible.ClickhouseRepo.aggregate(query, :count) + + {count == expected, count} + end, + 200, + 10 + ) + end end