Keep track of native stats start timestamp when retrieving data (#2715)
* Stats boundary/PoC? * Delete stats removal * Drop events check on site creation * Update seeds script * Use native_stats_start_at * Don't rely on native stats pointer in imported stats queries * Reset site * Export reset/1 * Remove unnecessary inserted_at settings * Update seeds * Remove unnecessary inserted_at settingpull/2719/head r2023.03.01-66
parent
05e7f93da2
commit
8f86036e57
@ -1,41 +1,14 @@
|
||||
defmodule Plausible.Site.Removal do
|
||||
@moduledoc """
|
||||
A service responsible for site and its stats deletion.
|
||||
The site deletion alone is done in postgres and is executed first,
|
||||
the latter deletions (events, sessions and imported tables in clickhouse)
|
||||
are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`.
|
||||
|
||||
This is to avoid race condition in which the site is deleted, but stats
|
||||
writes are pending (either in the buffers or are about to be buffered, due
|
||||
to Sites.Cache keeping the now obsolete record until refresh is triggered).
|
||||
A site deletion service stub.
|
||||
"""
|
||||
@stats_deletion_delay_seconds 60 * 20
|
||||
|
||||
alias Plausible.Workers.StatsRemoval
|
||||
alias Plausible.Repo
|
||||
alias Ecto.Multi
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
@spec stats_deletion_delay_seconds() :: pos_integer()
|
||||
def stats_deletion_delay_seconds() do
|
||||
@stats_deletion_delay_seconds
|
||||
end
|
||||
|
||||
@spec run(String.t()) :: {:ok, map()}
|
||||
def run(domain) do
|
||||
site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain
|
||||
|
||||
Multi.new()
|
||||
|> Multi.run(:site_id, fn _, _ ->
|
||||
{:ok, Repo.one(from s in site_by_domain_q, select: s.id)}
|
||||
end)
|
||||
|> Multi.delete_all(:delete_all, site_by_domain_q)
|
||||
|> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} ->
|
||||
StatsRemoval.new(%{domain: domain, site_id: site_id},
|
||||
schedule_in: stats_deletion_delay_seconds()
|
||||
)
|
||||
end)
|
||||
|> Repo.transaction()
|
||||
result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^domain))
|
||||
{:ok, %{delete_all: result}}
|
||||
end
|
||||
end
|
||||
|
||||
@ -1,59 +0,0 @@
|
||||
defmodule Plausible.Workers.StatsRemoval do
|
||||
@moduledoc """
|
||||
Asynchronous worker firing deletion mutations to clickhouse.
|
||||
For now only ALTER TABLE deletions are supported. Experimental
|
||||
DELETE FROM support is going to be introduced once production db
|
||||
is upgraded.
|
||||
|
||||
At most 3 attempts are made, with 15m backoff value.
|
||||
|
||||
Imported stats tables keep site reference through a numeric id, whilist
|
||||
events and sessions store domain as-is - hence two different deletes,
|
||||
one of which cannot be performed anymore once the site identifier is permanently
|
||||
gone from postgres.
|
||||
"""
|
||||
use Plausible.Repo
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :site_stats_removal,
|
||||
max_attempts: 3,
|
||||
unique: [period: :infinity, fields: [:args]]
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%{args: args}) do
|
||||
domain = Map.fetch!(args, "domain")
|
||||
site_id = Map.get(args, "site_id")
|
||||
|
||||
imported_result = delete_imported!(site_id)
|
||||
native_result = delete_native!(domain)
|
||||
|
||||
{:ok, Map.merge(imported_result, native_result)}
|
||||
end
|
||||
|
||||
@impl Oban.Worker
|
||||
def backoff(_job) do
|
||||
15 * 60
|
||||
end
|
||||
|
||||
defp delete_imported!(nil) do
|
||||
%{}
|
||||
end
|
||||
|
||||
defp delete_imported!(id) when is_integer(id) do
|
||||
Enum.map(Plausible.Imported.tables(), fn table ->
|
||||
sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"
|
||||
{table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])}
|
||||
end)
|
||||
|> Enum.into(%{})
|
||||
end
|
||||
|
||||
defp delete_native!(domain) do
|
||||
events_sql = "ALTER TABLE events DELETE WHERE domain = ?"
|
||||
sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"
|
||||
|
||||
%{
|
||||
"events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]),
|
||||
"sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain])
|
||||
}
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue