Add multiple imports per site (#3724)
* Clean up references to no longer active `google_analytics_imports` Oban queue * Stub CSV importer * Add SiteImport schema * Rename `Plausible.Imported` module file to match module name * Add `import_id` column to `Imported.*` CH schemas * Implement Importer behavior and manage imports state using new entities * Implement importer callbacks and maintain site.imported_data for UA * Keep imports in sync when forgetting all imports * Scope imported data queries to completed import IDs * Mark newly imported data with respective import ID * Clean up Importer implementation a bit * Test querying legacy and new imported data * Send Oban notifications on import worker failure too * Fix checking for forgettable imports and remove redundant function * Fix UA integration test * Change site import source to atom enum and add source label * Add typespecs and reduce repetition in `Plausible.Imported` * Improve documentation and typespecs * Add test for purging particular import * Switch email notification templates depending on import source * Document running import synchronously * Fix UA importer args parsing and ensure it's covered by tests * Clear `site.stats_start_date` on complete import to force recalculation * Test Oban notifications (h/t @ruslandoga) * Purge stats on import failure right away to reduce a chance of leaving debris behind * Fix typos Co-authored-by: hq1 <hq@mtod.org> * Fix another typo * Refactor fetching earliest import and earliest stats start date * Use `Date.after?` instead of `Timex.after?` * Cache import data in site virtual fields and limit queried imports to 5 * Ensure always current `stats_start_date` is used * Work around broken typespec in Timex * Make `SiteController.forget_imported` action idempotent * Discard irrecoverably failed import tasks * Use macros for site import statuses There's also a fix ensuring only complete imports are considered where relevant - couldn't isolate it as it was in a common hunk * Use `import_id` as worker job uniqueness criterion * Do not load imported stats data in plugins API context --------- Co-authored-by: hq1 <hq@mtod.org>pull/3721/head r2024.02.14-351
parent
6915dd5e78
commit
f8b4d5066a
@ -0,0 +1,110 @@
|
||||
defmodule Plausible.Imported do
|
||||
@moduledoc """
|
||||
Context for managing site statistics imports.
|
||||
|
||||
Currently following importers are implemented:
|
||||
|
||||
* `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google
|
||||
analytics formerly known as "Google Analytics"
|
||||
* `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes
|
||||
* `Plausible.Imported.CSVImporter` - a placeholder stub for CSV importer that will
|
||||
be added soon
|
||||
|
||||
For more information on implementing importers, see `Plausible.Imported.Importer`.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Imported.SiteImport
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
require Plausible.Imported.SiteImport
|
||||
|
||||
@tables [
|
||||
Imported.Visitor,
|
||||
Imported.Source,
|
||||
Imported.Page,
|
||||
Imported.EntryPage,
|
||||
Imported.ExitPage,
|
||||
Imported.Location,
|
||||
Imported.Device,
|
||||
Imported.Browser,
|
||||
Imported.OperatingSystem
|
||||
]
|
||||
|
||||
@table_names Enum.map(@tables, & &1.__schema__(:source))
|
||||
# Maximum number of complete imports to account for when querying stats
|
||||
@max_complete_imports 5
|
||||
|
||||
@spec tables() :: [String.t()]
|
||||
def tables, do: @table_names
|
||||
|
||||
@spec load_import_data(Site.t()) :: Site.t()
|
||||
def load_import_data(%{import_data_loaded: true} = site), do: site
|
||||
|
||||
def load_import_data(site) do
|
||||
complete_import_ids = list_complete_import_ids(site)
|
||||
earliest_import = get_earliest_import(site) || %{}
|
||||
|
||||
%{
|
||||
site
|
||||
| import_data_loaded: true,
|
||||
earliest_import_start_date: Map.get(earliest_import, :start_date),
|
||||
earliest_import_end_date: Map.get(earliest_import, :end_date),
|
||||
complete_import_ids: complete_import_ids
|
||||
}
|
||||
end
|
||||
|
||||
@spec list_all_imports(Site.t()) :: [SiteImport.t()]
|
||||
def list_all_imports(site) do
|
||||
from(i in SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
|
||||
def list_complete_import_ids(site) do
|
||||
ids =
|
||||
from(i in SiteImport,
|
||||
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
|
||||
select: i.id,
|
||||
limit: @max_complete_imports
|
||||
)
|
||||
|> Repo.all()
|
||||
|
||||
# account for legacy imports as well
|
||||
if site.imported_data && site.imported_data.status == "ok" do
|
||||
[0 | ids]
|
||||
else
|
||||
ids
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
|
||||
def get_earliest_import(site) do
|
||||
first_import =
|
||||
from(i in SiteImport,
|
||||
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
|
||||
order_by: i.start_date,
|
||||
limit: 1
|
||||
)
|
||||
|> Repo.one()
|
||||
|
||||
legacy_import =
|
||||
if site.imported_data && site.imported_data.status == "ok" do
|
||||
site.imported_data
|
||||
end
|
||||
|
||||
[legacy_import, first_import]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.min_by(& &1.start_date, Date, fn -> nil end)
|
||||
end
|
||||
|
||||
@spec delete_imports_for_site(Site.t()) :: :ok
|
||||
def delete_imports_for_site(site) do
|
||||
Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,25 @@
|
||||
defmodule Plausible.Imported.CSVImporter do
|
||||
@moduledoc """
|
||||
CSV importer stub.
|
||||
"""
|
||||
|
||||
use Plausible.Imported.Importer
|
||||
|
||||
@impl true
|
||||
def name(), do: :csv
|
||||
|
||||
@impl true
|
||||
def label(), do: "CSV"
|
||||
|
||||
# NOTE: change it once CSV import is implemented
|
||||
@impl true
|
||||
def email_template(), do: "google_analytics_import.html"
|
||||
|
||||
@impl true
|
||||
def parse_args(%{"s3_path" => s3_path}), do: [s3_path: s3_path]
|
||||
|
||||
@impl true
|
||||
def import_data(_site_import, _opts) do
|
||||
:ok
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,232 @@
|
||||
defmodule Plausible.Imported.Importer do
|
||||
@moduledoc """
|
||||
Behaviour that should be implemented for each import source.
|
||||
|
||||
All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`
|
||||
Oban worker. Each import source must define a module conforming `Importer` behaviour.
|
||||
|
||||
The callbacks that need to be implemented:
|
||||
|
||||
* `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.
|
||||
* `label/0` - Descriptive, display friendly name of the source.
|
||||
Example: "Google Analytics".
|
||||
* `email_template/0` - Name of the email template to use for notifications in
|
||||
`PlausibleWeb.Email` (`import_success` and `import_failure`). The template
|
||||
should have content customized for a particular source.
|
||||
* `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever
|
||||
options were passed to `new_import/3` will be present in the input map with string
|
||||
keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`
|
||||
is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.
|
||||
The expectation is parsing the map values producing a keyword list of options to
|
||||
pass to `import_data/2`.
|
||||
* `import_data/2` - Receives site import struct and options produced by `parse_args/1`.
|
||||
This is where all the import processing is done. The way the import is implemented
|
||||
is entirely arbitrary except the requirement that the process as a whole must
|
||||
by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`
|
||||
on successful import or `{:error, ...}` on failure. The map in success tuple is
|
||||
used for updating site import struct and is passed to `on_success/2` callback.
|
||||
Please note that error tuple should be only returned on errors that can't be
|
||||
recovered from. For transient errors, the import should throw an exception or
|
||||
simply crash.
|
||||
* `before_start/1` - Optional callback run right before scheduling import job. It's
|
||||
expected to either return `:ok` for the import to proceed or `{:error, ...}` tuple,
|
||||
which will be returned from `new_import/3` call.
|
||||
* `on_success/2` - Optional callback run once site import is completed. Receives map
|
||||
returned from `import_data/2`. Expected to always return `:ok`.
|
||||
* `on_failure/1` - Optional callback run when import job fails permanently.
|
||||
|
||||
All sources must be added to the list in `Plausible.Imported.ImportSources`.
|
||||
|
||||
In order to schedule a new import job using a given source, respective importer's
|
||||
`new_import/3` function must be called. It accepts site, user who is doing the import
|
||||
and any options necessary to carry out the import.
|
||||
|
||||
There's an expectation that `start_date` and `end_date` are provided either as options
|
||||
passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters
|
||||
are not provided, the import will eventually crash. These parameters define time range
|
||||
of imported data which is in turn used for efficient querying.
|
||||
|
||||
Logic running inside `import_data/2` is expected to populated all `imported_*` tables
|
||||
in ClickHouse with `import_id` column set to site import's ID.
|
||||
|
||||
Managing any configuration or authentication prior to running import is outside of
|
||||
scope of importer logic and is expected to be implemented separately.
|
||||
|
||||
## Running import fully synchronously
|
||||
|
||||
In case it's necessary to run the whole import job fully synchronously, the
|
||||
`Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message
|
||||
on completion, failure or transient failure of the import.
|
||||
|
||||
A basic usage scenario looks like this:
|
||||
|
||||
```elixir
|
||||
{:ok, job} = Plausible.Imported.NoopImporter.new_import(
|
||||
site,
|
||||
user,
|
||||
start_date: ~D[2005-01-01],
|
||||
end_date: Date.utc_today(),
|
||||
# this option is necessary to setup the calling process as listener
|
||||
listen?: true
|
||||
)
|
||||
|
||||
import_id = job.args[:import_id]
|
||||
|
||||
receive do
|
||||
{:notification, :analytics_imports_jobs, %{"complete" => ^import_id}} ->
|
||||
IO.puts("Job completed")
|
||||
|
||||
{:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}} ->
|
||||
IO.puts("Job failed transiently")
|
||||
|
||||
{:notification, :analytics_imports_jobs, %{"fail" => ^import_id}} ->
|
||||
IO.puts("Job failed permanently")
|
||||
after
|
||||
15_000 ->
|
||||
IO.puts("Job didn't finish in 15 seconds")
|
||||
end
|
||||
```
|
||||
|
||||
In a more realistic scenario, job scheduling will be done inside a GenServer process
|
||||
like LiveView, where notifications can be listened for via `handle_info/2`.
|
||||
"""
|
||||
|
||||
alias Plausible.Imported.SiteImport
|
||||
alias Plausible.Repo
|
||||
|
||||
@callback name() :: atom()
|
||||
@callback label() :: String.t()
|
||||
@callback email_template() :: String.t()
|
||||
@callback parse_args(map()) :: Keyword.t()
|
||||
@callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
|
||||
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
|
||||
@callback on_success(SiteImport.t(), map()) :: :ok
|
||||
@callback on_failure(SiteImport.t()) :: :ok
|
||||
|
||||
defmacro __using__(_opts) do
|
||||
quote do
|
||||
@behaviour Plausible.Imported.Importer
|
||||
|
||||
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
|
||||
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
|
||||
def new_import(site, user, opts) do
|
||||
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
|
||||
end
|
||||
|
||||
@doc false
|
||||
@spec run_import(SiteImport.t(), map()) :: :ok | {:error, :any}
|
||||
def run_import(site_import, args) do
|
||||
Plausible.Imported.Importer.run_import(
|
||||
site_import,
|
||||
args,
|
||||
&parse_args/1,
|
||||
&import_data/2,
|
||||
&on_success/2
|
||||
)
|
||||
end
|
||||
|
||||
@doc false
|
||||
@spec mark_failed(SiteImport.t()) :: SiteImport.t()
|
||||
def mark_failed(site_import) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.fail_changeset()
|
||||
|> Repo.update!()
|
||||
|
||||
:ok = on_failure(site_import)
|
||||
|
||||
site_import
|
||||
end
|
||||
|
||||
@impl true
|
||||
def before_start(_site_import), do: :ok
|
||||
|
||||
@impl true
|
||||
def on_success(_site_import, _extra_data), do: :ok
|
||||
|
||||
@impl true
|
||||
def on_failure(_site_import), do: :ok
|
||||
|
||||
defoverridable before_start: 1, on_success: 2, on_failure: 1
|
||||
end
|
||||
end
|
||||
|
||||
@doc false
|
||||
def new_import(source, site, user, opts, before_start_fun) do
|
||||
import_params =
|
||||
opts
|
||||
|> Keyword.take([:start_date, :end_date])
|
||||
|> Keyword.put(:source, source)
|
||||
|> Map.new()
|
||||
|
||||
Repo.transaction(fn ->
|
||||
result =
|
||||
site
|
||||
|> SiteImport.create_changeset(user, import_params)
|
||||
|> Repo.insert()
|
||||
|
||||
with {:ok, site_import} <- result,
|
||||
:ok <- before_start_fun.(site_import),
|
||||
{:ok, job} <- schedule_job(site_import, opts) do
|
||||
job
|
||||
else
|
||||
{:error, error} ->
|
||||
Repo.rollback(error)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.start_changeset()
|
||||
|> Repo.update!()
|
||||
|> Repo.preload(:site)
|
||||
|
||||
import_opts = parse_args_fun.(args)
|
||||
|
||||
case import_fun.(site_import, import_opts) do
|
||||
:ok ->
|
||||
{:ok, mark_complete(site_import, %{}, on_success_fun)}
|
||||
|
||||
{:ok, extra_data} ->
|
||||
{:ok, mark_complete(site_import, extra_data, on_success_fun)}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@oban_channel :analytics_imports_jobs
|
||||
|
||||
@doc false
|
||||
def notify(site_import, event) do
|
||||
Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})
|
||||
end
|
||||
|
||||
defp schedule_job(site_import, opts) do
|
||||
{listen?, opts} = Keyword.pop(opts, :listen?, false)
|
||||
|
||||
if listen? do
|
||||
:ok = Oban.Notifier.listen([@oban_channel])
|
||||
end
|
||||
|
||||
opts
|
||||
|> Keyword.put(:import_id, site_import.id)
|
||||
|> Map.new()
|
||||
|> Plausible.Workers.ImportAnalytics.new()
|
||||
|> Oban.insert()
|
||||
end
|
||||
|
||||
defp mark_complete(site_import, extra_data, on_success_fun) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.complete_changeset(extra_data)
|
||||
|> Repo.update!()
|
||||
|
||||
:ok = on_success_fun.(site_import, extra_data)
|
||||
|
||||
site_import
|
||||
end
|
||||
end
|
||||
@ -1,13 +0,0 @@
|
||||
defmodule Plausible.Imported do
|
||||
@tables ~w(
|
||||
imported_visitors imported_sources imported_pages imported_entry_pages
|
||||
imported_exit_pages imported_locations imported_devices imported_browsers
|
||||
imported_operating_systems
|
||||
)
|
||||
@spec tables() :: [String.t()]
|
||||
def tables, do: @tables
|
||||
|
||||
def forget(site) do
|
||||
Plausible.Purge.delete_imported_stats!(site)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,68 @@
|
||||
defmodule Plausible.Imported.SiteImport do
|
||||
@moduledoc """
|
||||
Site import schema.
|
||||
"""
|
||||
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
alias Plausible.Auth.User
|
||||
alias Plausible.Imported.ImportSources
|
||||
alias Plausible.Site
|
||||
|
||||
@statuses [:pending, :importing, :completed, :failed]
|
||||
|
||||
@type t() :: %__MODULE__{}
|
||||
|
||||
schema "site_imports" do
|
||||
field :start_date, :date
|
||||
field :end_date, :date
|
||||
field :source, Ecto.Enum, values: ImportSources.names()
|
||||
field :status, Ecto.Enum, values: @statuses
|
||||
|
||||
belongs_to :site, Site
|
||||
belongs_to :imported_by, User
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
for status <- @statuses do
|
||||
defmacro unquote(status)(), do: unquote(status)
|
||||
end
|
||||
|
||||
# NOTE: this is necessary for backwards compatibility
|
||||
# with legacy imports
|
||||
@spec label(t() | Site.ImportedData.t()) :: String.t()
|
||||
def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()
|
||||
def label(%Site.ImportedData{source: source}), do: source
|
||||
|
||||
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
|
||||
def create_changeset(site, user, params) do
|
||||
%__MODULE__{}
|
||||
|> cast(params, [:source, :start_date, :end_date])
|
||||
|> validate_required([:source])
|
||||
|> put_assoc(:site, site)
|
||||
|> put_assoc(:imported_by, user)
|
||||
|> put_change(:status, pending())
|
||||
end
|
||||
|
||||
@spec start_changeset(t()) :: Ecto.Changeset.t()
|
||||
def start_changeset(site_import) do
|
||||
site_import
|
||||
|> change(status: importing())
|
||||
end
|
||||
|
||||
@spec complete_changeset(t(), map()) :: Ecto.Changeset.t()
|
||||
def complete_changeset(site_import, params \\ %{}) do
|
||||
site_import
|
||||
|> cast(params, [:start_date, :end_date])
|
||||
|> put_change(:status, completed())
|
||||
|> validate_required([:start_date, :end_date])
|
||||
end
|
||||
|
||||
@spec fail_changeset(t()) :: Ecto.Changeset.t()
|
||||
def fail_changeset(site_import) do
|
||||
change(site_import, status: failed())
|
||||
end
|
||||
end
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue