Add multiple imports per site (#3724)

* Clean up references to no longer active `google_analytics_imports` Oban queue

* Stub CSV importer

* Add SiteImport schema

* Rename `Plausible.Imported` module file to match module name

* Add `import_id` column to `Imported.*` CH schemas

* Implement Importer behavior and manage imports state using new entities

* Implement importer callbacks and maintain site.imported_data for UA

* Keep imports in sync when forgetting all imports

* Scope imported data queries to completed import IDs

* Mark newly imported data with respective import ID

* Clean up Importer implementation a bit

* Test querying legacy and new imported data

* Send Oban notifications on import worker failure too

* Fix checking for forgettable imports and remove redundant function

* Fix UA integration test

* Change site import source to atom enum and add source label

* Add typespecs and reduce repetition in `Plausible.Imported`

* Improve documentation and typespecs

* Add test for purging particular import

* Switch email notification templates depending on import source

* Document running import synchronously

* Fix UA importer args parsing and ensure it's covered by tests

* Clear `site.stats_start_date` on complete import to force recalculation

* Test Oban notifications (h/t @ruslandoga)

* Purge stats on import failure right away to reduce a chance of leaving debris behind

* Fix typos

Co-authored-by: hq1 <hq@mtod.org>

* Fix another typo

* Refactor fetching earliest import and earliest stats start date

* Use `Date.after?` instead of `Timex.after?`

* Cache import data in site virtual fields and limit queried imports to 5

* Ensure always current `stats_start_date` is used

* Work around broken typespec in Timex

* Make `SiteController.forget_imported` action idempotent

* Discard irrecoverably failed import tasks

* Use macros for site import statuses

There's also a fix ensuring only complete imports are considered
where relevant - couldn't isolate it as it was in a common hunk

* Use `import_id` as worker job uniqueness criterion

* Do not load imported stats data in plugins API context

---------

Co-authored-by: hq1 <hq@mtod.org>
pull/3721/head r2024.02.14-351
Adrian Gruntkowski 3 years ago committed by GitHub
parent 6915dd5e78
commit f8b4d5066a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -524,8 +524,6 @@ base_queues = [
check_stats_emails: 1,
site_setup_emails: 1,
clean_invitations: 1,
# NOTE: to be removed once #3700 is released
google_analytics_imports: 1,
analytics_imports: 1,
domain_change_transition: 1,
check_accept_traffic_until: 1

@ -22,32 +22,31 @@ defmodule ObanErrorReporter do
Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
end
# NOTE: To be cleaned up once #3700 is released
@analytics_queues ["analytics_imports", "google_analytics_imports"]
defp on_job_exception(%Oban.Job{
queue: queue,
args: %{"site_id" => site_id, "source" => source},
queue: "analytics_imports",
args: %{"import_id" => import_id},
state: "executing",
attempt: attempt,
max_attempts: max_attempts
})
when queue in @analytics_queues and attempt >= max_attempts do
site = Plausible.Repo.get(Plausible.Site, site_id)
when attempt >= max_attempts do
site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
if site do
Plausible.Workers.ImportAnalytics.import_failed(source, site)
if site_import do
Plausible.Workers.ImportAnalytics.import_fail(site_import)
end
end
defp on_job_exception(%Oban.Job{
queue: queue,
args: %{"site_id" => site_id},
queue: "analytics_imports",
args: %{"import_id" => import_id},
state: "executing"
})
when queue in @analytics_queues do
site = Plausible.Repo.get(Plausible.Site, site_id)
Plausible.Purge.delete_imported_stats!(site)
}) do
site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
if site_import do
Plausible.Workers.ImportAnalytics.import_fail_transient(site_import)
end
end
defp on_job_exception(_job), do: :ignore

@ -0,0 +1,110 @@
defmodule Plausible.Imported do
@moduledoc """
Context for managing site statistics imports.
Currently following importers are implemented:
* `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google
analytics formerly known as "Google Analytics"
* `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes
* `Plausible.Imported.CSVImporter` - a placeholder stub for CSV importer that will
be added soon
For more information on implementing importers, see `Plausible.Imported.Importer`.
"""
import Ecto.Query
alias Plausible.Imported
alias Plausible.Imported.SiteImport
alias Plausible.Repo
alias Plausible.Site
require Plausible.Imported.SiteImport
@tables [
Imported.Visitor,
Imported.Source,
Imported.Page,
Imported.EntryPage,
Imported.ExitPage,
Imported.Location,
Imported.Device,
Imported.Browser,
Imported.OperatingSystem
]
@table_names Enum.map(@tables, & &1.__schema__(:source))
# Maximum number of complete imports to account for when querying stats
@max_complete_imports 5
@spec tables() :: [String.t()]
def tables, do: @table_names
@spec load_import_data(Site.t()) :: Site.t()
def load_import_data(%{import_data_loaded: true} = site), do: site
def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site)
earliest_import = get_earliest_import(site) || %{}
%{
site
| import_data_loaded: true,
earliest_import_start_date: Map.get(earliest_import, :start_date),
earliest_import_end_date: Map.get(earliest_import, :end_date),
complete_import_ids: complete_import_ids
}
end
@spec list_all_imports(Site.t()) :: [SiteImport.t()]
def list_all_imports(site) do
from(i in SiteImport, where: i.site_id == ^site.id)
|> Repo.all()
end
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
def list_complete_import_ids(site) do
ids =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: i.id,
limit: @max_complete_imports
)
|> Repo.all()
# account for legacy imports as well
if site.imported_data && site.imported_data.status == "ok" do
[0 | ids]
else
ids
end
end
@spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
def get_earliest_import(site) do
first_import =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
order_by: i.start_date,
limit: 1
)
|> Repo.one()
legacy_import =
if site.imported_data && site.imported_data.status == "ok" do
site.imported_data
end
[legacy_import, first_import]
|> Enum.reject(&is_nil/1)
|> Enum.min_by(& &1.start_date, Date, fn -> nil end)
end
@spec delete_imports_for_site(Site.t()) :: :ok
def delete_imports_for_site(site) do
Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
:ok
end
end

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Browser do
@primary_key false
schema "imported_browsers" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :browser, :string
field :visitors, Ch, type: "UInt64"

@ -0,0 +1,25 @@
defmodule Plausible.Imported.CSVImporter do
@moduledoc """
CSV importer stub.
"""
use Plausible.Imported.Importer
@impl true
def name(), do: :csv
@impl true
def label(), do: "CSV"
# NOTE: change it once CSV import is implemented
@impl true
def email_template(), do: "google_analytics_import.html"
@impl true
def parse_args(%{"s3_path" => s3_path}), do: [s3_path: s3_path]
@impl true
def import_data(_site_import, _opts) do
:ok
end
end

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Device do
@primary_key false
schema "imported_devices" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :device, :string
field :visitors, Ch, type: "UInt64"

@ -5,6 +5,7 @@ defmodule Plausible.Imported.EntryPage do
@primary_key false
schema "imported_entry_pages" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :entry_page, :string
field :visitors, Ch, type: "UInt64"

@ -5,6 +5,7 @@ defmodule Plausible.Imported.ExitPage do
@primary_key false
schema "imported_exit_pages" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :exit_page, :string
field :visitors, Ch, type: "UInt64"

@ -5,11 +5,18 @@ defmodule Plausible.Imported.ImportSources do
@sources [
Plausible.Imported.UniversalAnalytics,
Plausible.Imported.NoopImporter
Plausible.Imported.NoopImporter,
Plausible.Imported.CSVImporter
]
@sources_map Map.new(@sources, &{&1.name(), &1})
@source_names Enum.map(@sources, & &1.name())
@spec names() :: [atom()]
def names(), do: @source_names
@spec by_name(atom()) :: module()
def by_name(name) do
Map.fetch!(@sources_map, name)
end

@ -0,0 +1,232 @@
defmodule Plausible.Imported.Importer do
@moduledoc """
Behaviour that should be implemented for each import source.
All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`
Oban worker. Each import source must define a module conforming `Importer` behaviour.
The callbacks that need to be implemented:
* `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.
* `label/0` - Descriptive, display friendly name of the source.
Example: "Google Analytics".
* `email_template/0` - Name of the email template to use for notifications in
`PlausibleWeb.Email` (`import_success` and `import_failure`). The template
should have content customized for a particular source.
* `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever
options were passed to `new_import/3` will be present in the input map with string
keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`
is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.
The expectation is parsing the map values producing a keyword list of options to
pass to `import_data/2`.
* `import_data/2` - Receives site import struct and options produced by `parse_args/1`.
This is where all the import processing is done. The way the import is implemented
is entirely arbitrary except the requirement that the process as a whole must
by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`
on successful import or `{:error, ...}` on failure. The map in success tuple is
used for updating site import struct and is passed to `on_success/2` callback.
Please note that error tuple should be only returned on errors that can't be
recovered from. For transient errors, the import should throw an exception or
simply crash.
* `before_start/1` - Optional callback run right before scheduling import job. It's
expected to either return `:ok` for the import to proceed or `{:error, ...}` tuple,
which will be returned from `new_import/3` call.
* `on_success/2` - Optional callback run once site import is completed. Receives map
returned from `import_data/2`. Expected to always return `:ok`.
* `on_failure/1` - Optional callback run when import job fails permanently.
All sources must be added to the list in `Plausible.Imported.ImportSources`.
In order to schedule a new import job using a given source, respective importer's
`new_import/3` function must be called. It accepts site, user who is doing the import
and any options necessary to carry out the import.
There's an expectation that `start_date` and `end_date` are provided either as options
passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters
are not provided, the import will eventually crash. These parameters define time range
of imported data which is in turn used for efficient querying.
Logic running inside `import_data/2` is expected to populated all `imported_*` tables
in ClickHouse with `import_id` column set to site import's ID.
Managing any configuration or authentication prior to running import is outside of
scope of importer logic and is expected to be implemented separately.
## Running import fully synchronously
In case it's necessary to run the whole import job fully synchronously, the
`Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message
on completion, failure or transient failure of the import.
A basic usage scenario looks like this:
```elixir
{:ok, job} = Plausible.Imported.NoopImporter.new_import(
site,
user,
start_date: ~D[2005-01-01],
end_date: Date.utc_today(),
# this option is necessary to setup the calling process as listener
listen?: true
)
import_id = job.args[:import_id]
receive do
{:notification, :analytics_imports_jobs, %{"complete" => ^import_id}} ->
IO.puts("Job completed")
{:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}} ->
IO.puts("Job failed transiently")
{:notification, :analytics_imports_jobs, %{"fail" => ^import_id}} ->
IO.puts("Job failed permanently")
after
15_000 ->
IO.puts("Job didn't finish in 15 seconds")
end
```
In a more realistic scenario, job scheduling will be done inside a GenServer process
like LiveView, where notifications can be listened for via `handle_info/2`.
"""
alias Plausible.Imported.SiteImport
alias Plausible.Repo
@callback name() :: atom()
@callback label() :: String.t()
@callback email_template() :: String.t()
@callback parse_args(map()) :: Keyword.t()
@callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
@callback on_success(SiteImport.t(), map()) :: :ok
@callback on_failure(SiteImport.t()) :: :ok
defmacro __using__(_opts) do
quote do
@behaviour Plausible.Imported.Importer
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
def new_import(site, user, opts) do
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
end
@doc false
@spec run_import(SiteImport.t(), map()) :: :ok | {:error, :any}
def run_import(site_import, args) do
Plausible.Imported.Importer.run_import(
site_import,
args,
&parse_args/1,
&import_data/2,
&on_success/2
)
end
@doc false
@spec mark_failed(SiteImport.t()) :: SiteImport.t()
def mark_failed(site_import) do
site_import =
site_import
|> SiteImport.fail_changeset()
|> Repo.update!()
:ok = on_failure(site_import)
site_import
end
@impl true
def before_start(_site_import), do: :ok
@impl true
def on_success(_site_import, _extra_data), do: :ok
@impl true
def on_failure(_site_import), do: :ok
defoverridable before_start: 1, on_success: 2, on_failure: 1
end
end
@doc false
def new_import(source, site, user, opts, before_start_fun) do
import_params =
opts
|> Keyword.take([:start_date, :end_date])
|> Keyword.put(:source, source)
|> Map.new()
Repo.transaction(fn ->
result =
site
|> SiteImport.create_changeset(user, import_params)
|> Repo.insert()
with {:ok, site_import} <- result,
:ok <- before_start_fun.(site_import),
{:ok, job} <- schedule_job(site_import, opts) do
job
else
{:error, error} ->
Repo.rollback(error)
end
end)
end
@doc false
def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do
site_import =
site_import
|> SiteImport.start_changeset()
|> Repo.update!()
|> Repo.preload(:site)
import_opts = parse_args_fun.(args)
case import_fun.(site_import, import_opts) do
:ok ->
{:ok, mark_complete(site_import, %{}, on_success_fun)}
{:ok, extra_data} ->
{:ok, mark_complete(site_import, extra_data, on_success_fun)}
{:error, error} ->
{:error, error}
end
end
@oban_channel :analytics_imports_jobs
@doc false
def notify(site_import, event) do
Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})
end
defp schedule_job(site_import, opts) do
{listen?, opts} = Keyword.pop(opts, :listen?, false)
if listen? do
:ok = Oban.Notifier.listen([@oban_channel])
end
opts
|> Keyword.put(:import_id, site_import.id)
|> Map.new()
|> Plausible.Workers.ImportAnalytics.new()
|> Oban.insert()
end
defp mark_complete(site_import, extra_data, on_success_fun) do
site_import =
site_import
|> SiteImport.complete_changeset(extra_data)
|> Repo.update!()
:ok = on_success_fun.(site_import, extra_data)
site_import
end
end

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Location do
@primary_key false
schema "imported_locations" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :country, :string
field :region, :string

@ -3,20 +3,44 @@ defmodule Plausible.Imported.NoopImporter do
Stub import implementation.
"""
@name "Noop"
use Plausible.Imported.Importer
def name(), do: @name
@impl true
def name(), do: :noop
def create_job(site, opts) do
Plausible.Workers.ImportAnalytics.new(%{
"source" => @name,
"site_id" => site.id,
"error" => opts[:error]
})
end
@impl true
def label(), do: "Noop"
# reusing existing template from another source
@impl true
def email_template(), do: "google_analytics_import.html"
@impl true
def parse_args(opts), do: opts
def import(_site, %{"error" => true}), do: {:error, "Something went wrong"}
def import(_site, _opts), do: :ok
@impl true
def import_data(_site_import, %{"error" => true}), do: {:error, "Something went wrong"}
def import_data(_site_import, %{"crash" => true}), do: raise("boom")
def import_data(_site_import, _opts), do: :ok
@impl true
def before_start(site_import) do
send(self(), {:before_start, site_import.id})
:ok
end
@impl true
def on_success(site_import, _extra_data) do
send(self(), {:on_success, site_import.id})
:ok
end
@impl true
def on_failure(site_import) do
send(self(), {:on_failure, site_import.id})
:ok
end
end

@ -5,6 +5,7 @@ defmodule Plausible.Imported.OperatingSystem do
@primary_key false
schema "imported_operating_systems" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :operating_system, :string
field :visitors, Ch, type: "UInt64"

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Page do
@primary_key false
schema "imported_pages" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :hostname, :string
field :page, :string

@ -1,13 +0,0 @@
defmodule Plausible.Imported do
@tables ~w(
imported_visitors imported_sources imported_pages imported_entry_pages
imported_exit_pages imported_locations imported_devices imported_browsers
imported_operating_systems
)
@spec tables() :: [String.t()]
def tables, do: @tables
def forget(site) do
Plausible.Purge.delete_imported_stats!(site)
end
end

@ -0,0 +1,68 @@
defmodule Plausible.Imported.SiteImport do
@moduledoc """
Site import schema.
"""
use Ecto.Schema
import Ecto.Changeset
alias Plausible.Auth.User
alias Plausible.Imported.ImportSources
alias Plausible.Site
@statuses [:pending, :importing, :completed, :failed]
@type t() :: %__MODULE__{}
schema "site_imports" do
field :start_date, :date
field :end_date, :date
field :source, Ecto.Enum, values: ImportSources.names()
field :status, Ecto.Enum, values: @statuses
belongs_to :site, Site
belongs_to :imported_by, User
timestamps()
end
for status <- @statuses do
defmacro unquote(status)(), do: unquote(status)
end
# NOTE: this is necessary for backwards compatibility
# with legacy imports
@spec label(t() | Site.ImportedData.t()) :: String.t()
def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()
def label(%Site.ImportedData{source: source}), do: source
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
def create_changeset(site, user, params) do
%__MODULE__{}
|> cast(params, [:source, :start_date, :end_date])
|> validate_required([:source])
|> put_assoc(:site, site)
|> put_assoc(:imported_by, user)
|> put_change(:status, pending())
end
@spec start_changeset(t()) :: Ecto.Changeset.t()
def start_changeset(site_import) do
site_import
|> change(status: importing())
end
@spec complete_changeset(t(), map()) :: Ecto.Changeset.t()
def complete_changeset(site_import, params \\ %{}) do
site_import
|> cast(params, [:start_date, :end_date])
|> put_change(:status, completed())
|> validate_required([:start_date, :end_date])
end
@spec fail_changeset(t()) :: Ecto.Changeset.t()
def fail_changeset(site_import) do
change(site_import, status: failed())
end
end

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Source do
@primary_key false
schema "imported_sources" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :source, :string
field :utm_medium, :string

@ -3,50 +3,59 @@ defmodule Plausible.Imported.UniversalAnalytics do
Import implementation for Universal Analytics.
"""
use Plausible.ClickhouseRepo
use Plausible.Imported.Importer
alias Plausible.Site
alias Plausible.Repo
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@type job_opt() ::
{:view_id, non_neg_integer()}
| {:start_date | :end_date | :access_token | :refresh_token | :token_expires_at,
String.t()}
@impl true
def name(), do: :universal_analytics
@type import_opt() ::
{:view_id, non_neg_integer()}
| {:date_range, Date.Range.t()}
| {:auth, {String.t(), String.t(), String.t()}}
@impl true
def label(), do: "Google Analytics"
# NOTE: we have to use old name for now
@name "Google Analytics"
@impl true
def email_template(), do: "google_analytics_import.html"
@spec name() :: String.t()
def name(), do: @name
@impl true
def before_start(site_import) do
site = Repo.preload(site_import, :site).site
@spec create_job(Site.t(), [job_opt()]) :: Ecto.Changeset.t()
def create_job(site, opts) do
view_id = Keyword.fetch!(opts, :view_id)
start_date = Keyword.fetch!(opts, :start_date)
end_date = Keyword.fetch!(opts, :end_date)
access_token = Keyword.fetch!(opts, :access_token)
refresh_token = Keyword.fetch!(opts, :refresh_token)
token_expires_at = Keyword.fetch!(opts, :token_expires_at)
Plausible.Workers.ImportAnalytics.new(%{
"source" => @name,
"site_id" => site.id,
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token,
"refresh_token" => refresh_token,
"token_expires_at" => token_expires_at
})
site
|> Plausible.Site.start_import(
site_import.start_date,
site_import.end_date,
label()
)
|> Repo.update!()
:ok
end
@impl true
def on_success(site_import, _extra_data) do
site = Repo.preload(site_import, :site).site
site
|> Plausible.Site.import_success()
|> Repo.update!()
:ok
end
@impl true
def on_failure(site_import) do
site = Repo.preload(site_import, :site).site
site
|> Plausible.Site.import_failure()
|> Repo.update!()
:ok
end
@spec parse_args(map()) :: [import_opt()]
@impl true
def parse_args(
%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = args
) do
@ -73,8 +82,8 @@ defmodule Plausible.Imported.UniversalAnalytics do
This function fetches Google Analytics reports which are then passed in batches
to Clickhouse by the `Plausible.Imported.Buffer` process.
"""
@spec import(Site.t(), [import_opt()]) :: :ok | {:error, any()}
def import(site, opts) do
@impl true
def import_data(site_import, opts) do
date_range = Keyword.fetch!(opts, :date_range)
view_id = Keyword.fetch!(opts, :view_id)
auth = Keyword.fetch!(opts, :auth)
@ -82,7 +91,7 @@ defmodule Plausible.Imported.UniversalAnalytics do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
records = from_report(rows, site.id, table)
records = from_report(rows, site_import.site_id, site_import.id, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
@ -94,14 +103,14 @@ defmodule Plausible.Imported.UniversalAnalytics do
end
end
def from_report(nil, _site_id, _metric), do: nil
def from_report(nil, _site_id, _import_id, _metric), do: nil
def from_report(data, site_id, table) do
def from_report(data, site_id, import_id, table) do
Enum.reduce(data, [], fn row, acc ->
if Map.get(row.dimensions, "ga:date") in @missing_values do
acc
else
[new_from_report(site_id, table, row) | acc]
[new_from_report(site_id, import_id, table, row) | acc]
end
end)
end
@ -111,9 +120,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
round(float)
end
defp new_from_report(site_id, "imported_visitors", row) do
defp new_from_report(site_id, import_id, "imported_visitors", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("ga:pageviews") |> parse_number(),
@ -123,9 +133,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_sources", row) do
defp new_from_report(site_id, import_id, "imported_sources", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("ga:source") |> parse_referrer(),
utm_medium: row.dimensions |> Map.fetch!("ga:medium") |> default_if_missing(),
@ -139,9 +150,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_pages", row) do
defp new_from_report(site_id, import_id, "imported_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
hostname: row.dimensions |> Map.fetch!("ga:hostname") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("ga:pagePath") |> URI.parse() |> Map.get(:path),
@ -152,9 +164,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_entry_pages", row) do
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
entry_page: row.dimensions |> Map.fetch!("ga:landingPagePath"),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@ -164,9 +177,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_exit_pages", row) do
defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
exit_page: Map.fetch!(row.dimensions, "ga:exitPagePath"),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@ -174,13 +188,14 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_locations", row) do
defp new_from_report(site_id, import_id, "imported_locations", row) do
country_code = row.dimensions |> Map.fetch!("ga:countryIsoCode") |> default_if_missing("")
city_name = row.dimensions |> Map.fetch!("ga:city") |> default_if_missing("")
city_data = Location.get_city(city_name, country_code)
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
country: country_code,
region: row.dimensions |> Map.fetch!("ga:regionIsoCode") |> default_if_missing(""),
@ -192,9 +207,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
defp new_from_report(site_id, "imported_devices", row) do
defp new_from_report(site_id, import_id, "imported_devices", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
device: row.dimensions |> Map.fetch!("ga:deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@ -214,11 +230,12 @@ defmodule Plausible.Imported.UniversalAnalytics do
"(not set)" => ""
}
defp new_from_report(site_id, "imported_browsers", row) do
defp new_from_report(site_id, import_id, "imported_browsers", row) do
browser = Map.fetch!(row.dimensions, "ga:browser")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@ -234,11 +251,12 @@ defmodule Plausible.Imported.UniversalAnalytics do
"(not set)" => ""
}
defp new_from_report(site_id, "imported_operating_systems", row) do
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
os = Map.fetch!(row.dimensions, "ga:operatingSystem")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),

@ -5,6 +5,7 @@ defmodule Plausible.Imported.Visitor do
@primary_key false
schema "imported_visitors" do
field :site_id, Ch, type: "UInt64"
field :import_id, Ch, type: "UInt64"
field :date, :date
field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"

@ -33,18 +33,42 @@ defmodule Plausible.Purge do
- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)
"""
@spec delete_imported_stats!(Plausible.Site.t()) :: :ok
alias Plausible.Repo
@spec delete_imported_stats!(Plausible.Site.t() | Plausible.Imported.SiteImport.t()) :: :ok
@doc """
Deletes imported stats from Google Analytics, and clears the
`stats_start_date` field.
Deletes imported stats from and clears the `stats_start_date` field.
The `stats_start_date` is expected to get repopulated the next time
`Plausible.Sites.stats_start_date/1` is called.
If the input argument is a site, all imported stats are deleted. If it's a site import,
only imported stats for that import are deleted.
"""
def delete_imported_stats!(site) do
def delete_imported_stats!(%Plausible.Site{} = site) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])
end)
clear_stats_start_date!(site)
Plausible.Sites.clear_stats_start_date!(site)
:ok
end
def delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) do
site_import = Repo.preload(site_import, :site)
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"
Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [
site_import.site_id,
site_import.id
])
end)
Plausible.Sites.clear_stats_start_date!(site_import.site)
:ok
end
@ -67,10 +91,4 @@ defmodule Plausible.Purge do
)
|> Plausible.Repo.update!()
end
defp clear_stats_start_date!(site) do
site
|> Ecto.Changeset.change(stats_start_date: nil)
|> Plausible.Repo.update!()
end
end

@ -6,7 +6,6 @@ defmodule Plausible.Site do
import Ecto.Changeset
alias Plausible.Auth.User
alias Plausible.Site.GoogleAuth
alias Plausible.Timezones
@type t() :: %__MODULE__{}
@ -55,6 +54,14 @@ defmodule Plausible.Site do
field :entry_type, :string, virtual: true
field :pinned_at, :naive_datetime, virtual: true
# Used for caching imports data for the duration of the whole request
# to avoid multiple identical fetches. Populated by plugs putting
# `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true
field :earliest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps()
end
@ -159,7 +166,6 @@ defmodule Plausible.Site do
def import_success(site) do
change(site,
stats_start_date: site.imported_data.start_date,
imported_data: %{status: "ok"}
)
end
@ -168,50 +174,10 @@ defmodule Plausible.Site do
change(site, imported_data: %{status: "error"})
end
def set_imported_source(site, imported_source) do
change(site,
imported_data: %Plausible.Site.ImportedData{
end_date: Timex.today(),
source: imported_source
}
)
end
def remove_imported_data(site) do
change(site, imported_data: nil)
end
@doc """
Returns the date of the first recorded stat in the timezone configured by the user.
This function does 2 transformations:
UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date
## Examples
iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})
nil
iex> utc_start = ~N[2022-09-28 00:00:00]
iex> tz = "Europe/Helsinki"
iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
iex> Plausible.Site.local_start_date(site)
~D[2022-09-28]
iex> utc_start = ~N[2022-09-28 00:00:00]
iex> tz = "America/Los_Angeles"
iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
iex> Plausible.Site.local_start_date(site)
~D[2022-09-27]
"""
def local_start_date(%__MODULE__{stats_start_date: nil}) do
nil
end
def local_start_date(site) do
site.stats_start_date
|> Timezones.to_date_in_timezone(site.timezone)
end
defp clean_domain(changeset) do
clean_domain =
(get_field(changeset, :domain) || "")

@ -4,6 +4,8 @@ defmodule Plausible.Site.ImportedData do
"""
use Ecto.Schema
@type t() :: %__MODULE__{}
embedded_schema do
field :start_date, :date
field :end_date, :date

@ -189,13 +189,50 @@ defmodule Plausible.Sites do
end
end
@spec clear_stats_start_date!(Site.t()) :: Site.t()
def clear_stats_start_date!(site) do
site
|> Ecto.Changeset.change(stats_start_date: nil)
|> Plausible.Repo.update!()
end
@doc """
Returns the date of the first recorded stat in the timezone configured by the user.
This function does 2 transformations:
UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date
## Examples
iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})
nil
iex> utc_start = ~N[2022-09-28 00:00:00]
iex> tz = "Europe/Helsinki"
iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
iex> Plausible.Site.local_start_date(site)
~D[2022-09-28]
iex> utc_start = ~N[2022-09-28 00:00:00]
iex> tz = "America/Los_Angeles"
iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
iex> Plausible.Site.local_start_date(site)
~D[2022-09-27]
"""
@spec local_start_date(Site.t()) :: Date.t() | nil
def local_start_date(site) do
if stats_start_date = stats_start_date(site) do
Plausible.Timezones.to_date_in_timezone(stats_start_date, site.timezone)
end
end
@spec stats_start_date(Site.t()) :: Date.t() | nil
@doc """
Returns the date of the first event of the given site, or `nil` if the site
does not have stats yet.
If this is the first time the function is called for the site, it queries
Clickhouse and saves the date in the sites table.
imported stats and Clickhouse, choosing the earliest start date and saves
it in the sites table.
"""
def stats_start_date(site)
@ -204,7 +241,17 @@ defmodule Plausible.Sites do
end
def stats_start_date(%Site{} = site) do
if start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) do
site = Plausible.Imported.load_import_data(site)
start_date =
[
site.earliest_import_start_date,
Plausible.Stats.Clickhouse.pageview_start_date_local(site)
]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
if start_date do
updated_site =
site
|> Site.set_stats_start_date(start_date)

@ -16,9 +16,12 @@ defmodule Plausible.Stats.Imported do
query,
metrics
) do
import_ids = site.complete_import_ids
imported_q =
from(v in "imported_visitors",
where: v.site_id == ^site.id,
where: v.import_id in ^import_ids,
where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,
select: %{}
)
@ -107,11 +110,14 @@ defmodule Plausible.Stats.Imported do
{"imported_#{dim}s", String.to_existing_atom(dim)}
end
import_ids = site.complete_import_ids
imported_q =
from(
i in table,
group_by: field(i, ^dim),
where: i.site_id == ^site.id,
where: i.import_id in ^import_ids,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.visitors > 0,
select: %{}
@ -315,10 +321,13 @@ defmodule Plausible.Stats.Imported do
end
def merge_imported(q, site, query, :aggregate, metrics) do
import_ids = site.complete_import_ids
imported_q =
from(
i in "imported_visitors",
where: i.site_id == ^site.id,
where: i.import_id in ^import_ids,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
select: %{}
)

@ -78,7 +78,7 @@ defmodule Plausible.Stats.Interval do
@valid_by_period
end
with %Date{} = stats_start <- site.stats_start_date,
with %Date{} = stats_start <- Plausible.Sites.stats_start_date(site),
true <- abs(Timex.diff(Date.utc_today(), stats_start, :months)) > 12 do
Map.replace(table, "all", ["week", "month"])
else

@ -121,7 +121,7 @@ defmodule Plausible.Stats.Query do
defp put_period(query, site, %{"period" => "all"}) do
now = today(site.timezone)
start_date = Plausible.Site.local_start_date(site) || now
start_date = Plausible.Sites.local_start_date(site) || now
struct!(query,
period: "all",
@ -251,9 +251,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do
cond do
is_nil(site.imported_data) -> false
site.imported_data.status != "ok" -> false
Timex.after?(query.date_range.first, site.imported_data.end_date) -> false
is_nil(site.earliest_import_end_date) -> false
Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
Enum.any?(query.filters) -> false
query.period == "realtime" -> false
true -> requested?

@ -22,7 +22,14 @@ defmodule Plausible.Timezones do
@spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t()
def to_date_in_timezone(dt, timezone) do
utc_dt = Timex.Timezone.convert(dt, "UTC")
utc_dt =
case dt do
%Date{} ->
Timex.to_datetime(dt, "UTC")
dt ->
Timex.Timezone.convert(dt, "UTC")
end
case Timex.Timezone.convert(utc_dt, timezone) do
%DateTime{} = tz_dt ->

@ -3,6 +3,8 @@ defmodule PlausibleWeb.Api.StatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Imported.SiteImport
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
@ -132,6 +134,8 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
plot: plot_timeseries(timeseries_result, selected_metric),
labels: labels,
@ -140,7 +144,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: with_imported?(query, comparison_query),
imported_source: site.imported_data && site.imported_data.source,
imported_source: site_import && SiteImport.label(site_import),
full_intervals: full_intervals
})
else
@ -216,12 +220,14 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query),
imported_source: site.imported_data && site.imported_data.source,
imported_source: site_import && SiteImport.label(site_import),
comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first,

@ -764,11 +764,12 @@ defmodule PlausibleWeb.SiteController do
"expires_at" => expires_at
}) do
site = conn.assigns.site
source = "Google Analytics"
current_user = conn.assigns.current_user
job =
Plausible.Imported.UniversalAnalytics.create_job(
{:ok, _} =
Plausible.Imported.UniversalAnalytics.new_import(
site,
current_user,
view_id: view_id,
start_date: start_date,
end_date: end_date,
@ -777,49 +778,46 @@ defmodule PlausibleWeb.SiteController do
token_expires_at: expires_at
)
Ecto.Multi.new()
|> Ecto.Multi.update(
:update_site,
Plausible.Site.start_import(site, start_date, end_date, source)
)
|> Oban.insert(:oban_job, job)
|> Repo.transaction()
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
# NOTE: To be cleaned up once #3700 is released
@analytics_queues ["analytics_imports", "google_analytics_imports"]
def forget_imported(conn, _params) do
site = conn.assigns[:site]
site = conn.assigns.site
cond do
site.imported_data ->
Oban.cancel_all_jobs(
from j in Oban.Job,
where:
j.queue in @analytics_queues and
fragment("(? ->> 'site_id')::int", j.args) == ^site.id
)
import_ids =
site
|> Plausible.Imported.list_all_imports()
|> Enum.map(& &1.id)
import_ids =
if site.imported_data do
[0 | import_ids]
else
import_ids
end
Plausible.Imported.forget(site)
if import_ids != [] do
Oban.cancel_all_jobs(
from j in Oban.Job,
where:
j.queue == "analytics_imports" and
fragment("(? ->> 'import_id')::int", j.args) in ^import_ids
)
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
Plausible.Purge.delete_imported_stats!(site)
conn
|> put_flash(:success, "Imported data has been cleared")
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
Plausible.Imported.delete_imports_for_site(site)
true ->
conn
|> put_flash(:error, "No data has been imported")
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()
end
conn
|> put_flash(:success, "Imported data has been cleared")
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
def change_domain(conn, _params) do

@ -309,6 +309,7 @@ defmodule PlausibleWeb.StatsController do
cond do
!shared_link.site.locked ->
shared_link = Plausible.Repo.preload(shared_link, site: :owner)
stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)
conn
|> put_resp_header("x-robots-tag", "noindex, nofollow")
@ -318,7 +319,7 @@ defmodule PlausibleWeb.StatsController do
has_goals: Sites.has_goals?(shared_link.site),
funnels: list_funnels(shared_link.site),
has_props: Plausible.Props.configured?(shared_link.site),
stats_start_date: shared_link.site.stats_start_date,
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: title(conn, shared_link.site),
offer_email_report: false,

@ -313,29 +313,35 @@ defmodule PlausibleWeb.Email do
)
end
# NOTE: make email different depending on import source
def import_success(source, user, site) do
def import_success(site_import, user) do
import_api = Plausible.Imported.ImportSources.by_name(site_import.source)
label = import_api.label()
priority_email()
|> to(user)
|> tag("import-success-email")
|> subject("#{source} data imported for #{site.domain}")
|> render("google_analytics_import.html", %{
site: site,
link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain),
|> subject("#{label} data imported for #{site_import.site.domain}")
|> render(import_api.email_template(), %{
site_import: site_import,
label: label,
link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site_import.site.domain),
user: user,
success: true
})
end
# NOTE: make email different depending on import source
def import_failure(source, user, site) do
def import_failure(site_import, user) do
import_api = Plausible.Imported.ImportSources.by_name(site_import.source)
label = import_api.label()
priority_email()
|> to(user)
|> tag("import-failure-email")
|> subject("#{source} import failed for #{site.domain}")
|> render("google_analytics_import.html", %{
|> subject("#{label} import failed for #{site_import.site.domain}")
|> render(import_api.email_template(), %{
site_import: site_import,
label: label,
user: user,
site: site,
success: false
})
end

@ -47,6 +47,8 @@ defmodule PlausibleWeb.AuthorizeSiteAccess do
Sentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})
Plausible.OpenTelemetry.add_site_attributes(site)
site = Plausible.Imported.load_import_data(site)
merge_assigns(conn, site: site, current_user_role: role)
else
PlausibleWeb.ControllerHelpers.render_error(conn, 404) |> halt

@ -16,6 +16,7 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug do
:ok <- check_api_key_rate_limit(api_key),
{:ok, site} <- verify_access(api_key, conn.params["site_id"]) do
Plausible.OpenTelemetry.add_site_attributes(site)
site = Plausible.Imported.load_import_data(site)
assign(conn, :site, site)
else
{:error, :missing_api_key} ->

@ -1,11 +1,11 @@
<%= if @success do %>
Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site.domain %> now contains historical imported data from <%= date_format(
@site.imported_data.start_date
) %> to <%= date_format(@site.imported_data.end_date) %>
Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site_import.site.domain %> now contains historical imported data from <%= date_format(
@site_import.start_date
) %> to <%= date_format(@site_import.end_date) %>
<br /><br />
<%= link("Click here", to: @link) %> to view your dashboard.
<% else %>
Unfortunately, your Google Analytics import for <%= @site.domain %> did not complete successfully. Sorry about that!
Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
<br /> <br />
This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
<br /> <br />

@ -9,33 +9,41 @@ defmodule Plausible.Workers.ImportAnalytics do
use Oban.Worker,
queue: :analytics_imports,
max_attempts: 3,
unique: [fields: [:args], period: 60]
unique: [fields: [:args], keys: [:import_id], period: 60]
alias Plausible.Imported.ImportSources
alias Plausible.Imported.Importer
alias Plausible.Imported.SiteImport
@impl Oban.Worker
def perform(%Oban.Job{
args: %{"site_id" => site_id, "source" => source} = args
args: %{"import_id" => import_id} = args
}) do
import_api = ImportSources.by_name(source)
import_opts = import_api.parse_args(args)
site_import =
SiteImport
|> Repo.get!(import_id)
|> Repo.preload(:site)
site = Repo.get!(Plausible.Site, site_id)
import_api = ImportSources.by_name(site_import.source)
case import_api.import(site, import_opts) do
:ok ->
import_success(source, site)
case import_api.run_import(site_import, args) do
{:ok, site_import} ->
import_complete(site_import)
:ok
{:error, error} ->
Sentry.capture_message("Failed to import from Google Analytics",
extra: %{site: site.domain, error: inspect(error)}
Sentry.capture_message("Failed to import from #{site_import.source}",
extra: %{
import_id: site_import.id,
site: site_import.site.domain,
error: inspect(error)
}
)
import_failed(source, site)
import_fail(site_import)
{:error, error}
{:discard, error}
end
end
@ -45,33 +53,44 @@ defmodule Plausible.Workers.ImportAnalytics do
300
end
def import_success(source, site) do
site = Repo.preload(site, memberships: :user)
def import_complete(site_import) do
site_import = Repo.preload(site_import, site: [memberships: :user])
site
|> Plausible.Site.import_success()
|> Repo.update!()
Enum.each(site.memberships, fn membership ->
Enum.each(site_import.site.memberships, fn membership ->
if membership.role in [:owner, :admin] do
PlausibleWeb.Email.import_success(source, membership.user, site)
PlausibleWeb.Email.import_success(site_import, membership.user)
|> Plausible.Mailer.send()
end
end)
Plausible.Sites.clear_stats_start_date!(site_import.site)
Importer.notify(site_import, :complete)
:ok
end
def import_failed(source, site) do
site = Repo.preload(site, memberships: :user)
def import_fail_transient(site_import) do
Plausible.Purge.delete_imported_stats!(site_import)
Importer.notify(site_import, :transient_fail)
end
def import_fail(site_import) do
Plausible.Purge.delete_imported_stats!(site_import)
import_api = ImportSources.by_name(site_import.source)
site
|> Plausible.Site.import_failure()
|> Repo.update!()
site_import =
site_import
|> import_api.mark_failed()
|> Repo.preload(site: [memberships: :user])
Plausible.Purge.delete_imported_stats!(site)
Importer.notify(site_import, :fail)
Enum.each(site.memberships, fn membership ->
Enum.each(site_import.site.memberships, fn membership ->
if membership.role in [:owner, :admin] do
PlausibleWeb.Email.import_failure(source, membership.user, site)
PlausibleWeb.Email.import_failure(site_import, membership.user)
|> Plausible.Mailer.send()
end
end)

@ -40,7 +40,7 @@ defmodule Plausible.Google.ApiTest do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
records = UniversalAnalytics.from_report(rows, site.id, table)
records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
@ -82,7 +82,7 @@ defmodule Plausible.Google.ApiTest do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
records = UniversalAnalytics.from_report(rows, site.id, table)
records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end

File diff suppressed because it is too large Load Diff

@ -2,41 +2,58 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
use Plausible.DataCase, async: true
use Plausible.Test.Support.HTTPMocker
alias Plausible.Imported.SiteImport
alias Plausible.Imported.UniversalAnalytics
require Plausible.Imported.SiteImport
setup [:create_user, :create_new_site]
describe "create_job/2 and parse_args/1" do
test "parses job args properly" do
site = insert(:site)
site_id = site.id
describe "new_import/3 and parse_args/1" do
test "parses job args properly", %{user: user, site: site} do
expires_at = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
job =
UniversalAnalytics.create_job(site,
view_id: 123,
start_date: "2023-10-01",
end_date: "2024-01-02",
access_token: "access123",
refresh_token: "refresh123",
token_expires_at: expires_at
)
assert {:ok, job} =
UniversalAnalytics.new_import(site, user,
view_id: 123,
start_date: "2023-10-01",
end_date: "2024-01-02",
access_token: "access123",
refresh_token: "refresh123",
token_expires_at: expires_at
)
assert %Oban.Job{
args:
%{
"import_id" => import_id,
"view_id" => 123,
"start_date" => "2023-10-01",
"end_date" => "2024-01-02",
"access_token" => "access123",
"refresh_token" => "refresh123",
"token_expires_at" => ^expires_at
} = args
} = Repo.reload!(job)
assert [
%{
id: ^import_id,
source: :universal_analytics,
start_date: ~D[2023-10-01],
end_date: ~D[2024-01-02],
status: SiteImport.pending()
}
] = Plausible.Imported.list_all_imports(site)
assert %Ecto.Changeset{
data: %Oban.Job{},
changes: %{
args:
%{
"site_id" => ^site_id,
"view_id" => 123,
"start_date" => "2023-10-01",
"end_date" => "2024-01-02",
"access_token" => "access123",
"refresh_token" => "refresh123",
"token_expires_at" => ^expires_at
} = args
assert %{
imported_data: %{
source: "Google Analytics",
start_date: ~D[2023-10-01],
end_date: ~D[2024-01-02],
status: "importing"
}
} = job
} = Repo.reload!(site)
assert opts = [_ | _] = UniversalAnalytics.parse_args(args)
@ -46,23 +63,34 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
end
end
describe "import/2" do
describe "import_data/2" do
@tag :slow
test "imports page views from Google Analytics", %{site: site} do
test "imports page views from Google Analytics", %{user: user, site: site} do
mock_http_with("google_analytics_import#1.json")
view_id = "54297898"
date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
start_date = ~D[2011-01-01]
end_date = ~D[2022-07-19]
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()
auth = {"***", "refresh_token", future}
access_token = "***"
refresh_token = "refresh_token"
assert :ok ==
UniversalAnalytics.import(site,
date_range: date_range,
view_id: view_id,
auth: auth
)
{:ok, job} =
UniversalAnalytics.new_import(
site,
user,
view_id: view_id,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: future
)
job = Repo.reload!(job)
Plausible.Workers.ImportAnalytics.perform(job)
assert 1_495_150 == Plausible.Stats.Clickhouse.imported_pageview_count(site)
end

@ -2,9 +2,28 @@ defmodule Plausible.PurgeTest do
use Plausible.DataCase
setup do
site = insert(:site, stats_start_date: ~D[2020-01-01])
user = insert(:user)
site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])
populate_stats(site, [
import_params = %{
source: :universal_analytics,
start_date: ~D[2005-01-01],
end_date: Timex.today()
}
[site_import1, site_import2] =
Enum.map(1..2, fn _ ->
site
|> Plausible.Imported.SiteImport.create_changeset(
user,
import_params
)
|> Plausible.Repo.insert!()
|> Plausible.Imported.SiteImport.complete_changeset()
|> Plausible.Repo.update!()
end)
populate_stats(site, site_import1.id, [
build(:pageview),
build(:imported_visitors),
build(:imported_sources),
@ -17,13 +36,26 @@ defmodule Plausible.PurgeTest do
build(:imported_operating_systems)
])
{:ok, %{site: site}}
populate_stats(site, site_import2.id, [
build(:pageview),
build(:imported_visitors),
build(:imported_sources),
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
build(:imported_operating_systems)
])
{:ok, %{site: site, site_import1: site_import1, site_import2: site_import2}}
end
test "delete_imported_stats!/1 deletes imported data", %{site: site} do
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert await_clickhouse_count(query, 1)
assert await_clickhouse_count(query, 2)
end)
assert :ok == Plausible.Purge.delete_imported_stats!(site)
@ -34,6 +66,29 @@ defmodule Plausible.PurgeTest do
end)
end
test "delete_imported_stats!/1 deletes imported data only for a particular site import", %{
site_import1: site_import1,
site_import2: site_import2
} do
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.import_id == ^site_import1.id)
assert await_clickhouse_count(query, 1)
query = from(imported in table, where: imported.import_id == ^site_import2.id)
assert await_clickhouse_count(query, 1)
end)
assert :ok == Plausible.Purge.delete_imported_stats!(site_import1)
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.import_id == ^site_import1.id)
assert await_clickhouse_count(query, 0)
query = from(imported in table, where: imported.import_id == ^site_import2.id)
assert await_clickhouse_count(query, 1)
end)
end
test "delete_imported_stats!/1 resets stats_start_date", %{site: site} do
assert :ok == Plausible.Purge.delete_imported_stats!(site)
assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)

@ -72,7 +72,7 @@ defmodule Plausible.Stats.IntervalTest do
describe "valid_for_period/3" do
test "common" do
site = build(:site)
site = insert(:site)
assert valid_for_period?("month", "date", site: site)
refute valid_for_period?("30d", "month", site: site)
refute valid_for_period?("realtime", "week", site: site)

@ -120,7 +120,7 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows hourly if site is completely new", %{site: site} do
site = Map.put(site, :stats_start_date, Timex.now())
site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date())
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today()
@ -130,7 +130,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows daily if site is more than a day old", %{site: site} do
site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1))
site =
Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date())
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today() |> Timex.shift(days: -1)
@ -140,7 +142,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows monthly if site is more than a month old", %{site: site} do
site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))
site =
Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)
@ -150,7 +154,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time uses passed interval different from the default interval", %{site: site} do
site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))
site =
Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
q = Query.from(site, %{"period" => "all", "interval" => "week"})
assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)

@ -8,6 +8,10 @@ defmodule PlausibleWeb.SiteControllerTest do
import Mox
import Plausible.Test.Support.HTML
alias Plausible.Imported.SiteImport
require Plausible.Imported.SiteImport
@v4_business_plan_id "857105"
setup :verify_on_exit!
@ -1260,7 +1264,7 @@ defmodule PlausibleWeb.SiteControllerTest do
describe "POST /:website/settings/google-import" do
setup [:create_user, :log_in, :create_new_site]
test "adds in-progress imported tag to site", %{conn: conn, site: site} do
test "creates site import instance", %{conn: conn, site: site} do
post(conn, "/#{site.domain}/settings/google-import", %{
"view_id" => "123",
"start_date" => "2018-03-01",
@ -1270,12 +1274,11 @@ defmodule PlausibleWeb.SiteControllerTest do
"expires_at" => "2022-09-22T20:01:37.112777"
})
imported_data = Repo.reload(site).imported_data
[site_import] = Plausible.Imported.list_all_imports(site)
assert imported_data
assert imported_data.source == "Google Analytics"
assert imported_data.end_date == ~D[2022-03-01]
assert imported_data.status == "importing"
assert site_import.source == :universal_analytics
assert site_import.end_date == ~D[2022-03-01]
assert site_import.status == SiteImport.pending()
end
test "schedules an import job in Oban", %{conn: conn, site: site} do
@ -1288,11 +1291,12 @@ defmodule PlausibleWeb.SiteControllerTest do
"expires_at" => "2022-09-22T20:01:37.112777"
})
assert [%{id: import_id}] = Plausible.Imported.list_all_imports(site)
assert_enqueued(
worker: Plausible.Workers.ImportAnalytics,
args: %{
"source" => "Google Analytics",
"site_id" => site.id,
"import_id" => import_id,
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
@ -1313,9 +1317,13 @@ defmodule PlausibleWeb.SiteControllerTest do
assert Repo.reload(site).imported_data == nil
end
test "removes actual imported data from Clickhouse", %{conn: conn, site: site} do
Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")
|> Repo.update!()
test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} do
Plausible.Imported.NoopImporter.new_import(
site,
user,
start_date: ~D[2022-01-01],
end_date: Timex.today()
)
populate_stats(site, [
build(:imported_visitors, pageviews: 10)
@ -1329,20 +1337,14 @@ defmodule PlausibleWeb.SiteControllerTest do
end)
end
test "cancels Oban job if it exists", %{conn: conn, site: site} do
test "cancels Oban job if it exists", %{conn: conn, user: user, site: site} do
{:ok, job} =
Plausible.Workers.ImportAnalytics.new(%{
"source" => "Google Analytics",
"site_id" => site.id,
"view_id" => "123",
"start_date" => "2022-01-01",
"end_date" => "2023-01-01",
"access_token" => "token"
})
|> Oban.insert()
Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")
|> Repo.update!()
Plausible.Imported.NoopImporter.new_import(
site,
user,
start_date: ~D[2022-01-01],
end_date: Timex.today()
)
populate_stats(site, [
build(:imported_visitors, pageviews: 10)

@ -145,6 +145,21 @@ defmodule Plausible.TestUtils do
:ok
end
def populate_stats(site, import_id, events) do
Enum.map(events, fn event ->
event = Map.put(event, :site_id, site.id)
case event do
%Plausible.ClickhouseEventV2{} ->
event
imported_event ->
Map.put(imported_event, :import_id, import_id)
end
end)
|> populate_stats
end
def populate_stats(site, events) do
Enum.map(events, fn event ->
Map.put(event, :site_id, site.id)

@ -2,64 +2,77 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
use Plausible.DataCase
use Bamboo.Test
alias Plausible.Imported.SiteImport
alias Plausible.Workers.ImportAnalytics
require Plausible.Imported.SiteImport
@moduletag capture_log: true
describe "Universal Analytics" do
describe "perform/1" do
setup do
%{
imported_data: %Plausible.Site.ImportedData{
import_opts: [
start_date: Timex.today() |> Timex.shift(days: -7),
end_date: Timex.today(),
source: "Noop",
status: "importing"
}
end_date: Timex.today()
]
}
end
test "updates the imported_data field for the site after successful import", %{
imported_data: imported_data
test "updates site import after successful import", %{
import_opts: import_opts
} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user])
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
assert [%{status: SiteImport.pending()}] = Plausible.Imported.list_all_imports(site)
# before_start callback triggered
assert_received {:before_start, import_id}
assert :ok =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id
}
})
assert [%{id: ^import_id, status: SiteImport.completed()}] =
Plausible.Imported.list_all_imports(site)
assert Repo.reload!(site).imported_data.status == "ok"
# on_success callback triggered
assert_received {:on_success, ^import_id}
end
test "updates the stats_start_date field for the site after successful import", %{
imported_data: imported_data
test "clears stats_start_date field for the site after successful import", %{
import_opts: import_opts
} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id
}
})
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
assert Repo.reload!(site).stats_start_date == imported_data.start_date
assert :ok =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
site = Repo.reload!(site)
assert site.stats_start_date == nil
assert Plausible.Sites.stats_start_date(site) == import_opts[:start_date]
assert Repo.reload!(site).stats_start_date == import_opts[:start_date]
end
test "sends email to owner after successful import", %{imported_data: imported_data} do
test "sends email to owner after successful import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user])
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id
}
})
assert :ok =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert_email_delivered_with(
to: [user],
@ -67,54 +80,57 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
)
end
test "updates site record after failed import", %{imported_data: imported_data} do
test "updates site import record after failed import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user])
import_opts = Keyword.put(import_opts, :error, true)
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id,
"error" => true
}
})
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
assert Repo.reload!(site).imported_data.status == "error"
assert {:discard, "Something went wrong"} =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert [%{status: SiteImport.failed()}] = Plausible.Imported.list_all_imports(site)
end
test "clears any orphaned data during import", %{imported_data: imported_data} do
test "clears any orphaned data during import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user])
import_opts = Keyword.put(import_opts, :error, true)
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
populate_stats(site, [
build(:imported_visitors, pageviews: 10)
build(:imported_visitors, import_id: job.args.import_id, pageviews: 10)
])
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id,
"error" => true
}
})
assert {:discard, _} =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
{count == 0, count}
end)
# on_failure callback triggered
assert_received {:on_failure, _import_id}
end
test "sends email to owner after failed import", %{imported_data: imported_data} do
test "sends email to owner after failed import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user], imported_data: imported_data)
site = insert(:site, members: [user])
import_opts = Keyword.put(import_opts, :error, true)
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
ImportAnalytics.perform(%Oban.Job{
args: %{
"source" => "Noop",
"site_id" => site.id,
"error" => true
}
})
assert {:discard, _} =
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert_email_delivered_with(
to: [user],
@ -122,4 +138,96 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
)
end
end
describe "perform/1 notifications" do
setup do
on_exit(fn ->
Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
Repo.delete_all(Plausible.Site)
Repo.delete_all(Plausible.Auth.User)
end)
:ok
end)
%{
import_opts: [
start_date: Timex.today() |> Timex.shift(days: -7),
end_date: Timex.today()
]
}
end
test "sends oban notification to calling process on completion when instructed", %{
import_opts: import_opts
} do
Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user])
import_opts = Keyword.put(import_opts, :listen?, true)
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
import_id = job.args[:import_id]
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert_receive {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}}
end)
end
test "sends oban notification to calling process on permanent failure when instructed", %{
import_opts: import_opts
} do
Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user])
import_opts =
import_opts
|> Keyword.put(:listen?, true)
|> Keyword.put(:error, true)
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
import_id = job.args[:import_id]
job
|> Repo.reload!()
|> ImportAnalytics.perform()
assert_receive {:notification, :analytics_imports_jobs, %{"fail" => ^import_id}}
end)
end
test "sends oban notification to calling process on transient failure when instructed", %{
import_opts: import_opts
} do
Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
site = insert(:site, members: [user])
import_opts =
import_opts
|> Keyword.put(:listen?, true)
|> Keyword.put(:crash, true)
{:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
import_id = job.args[:import_id]
site_import = Repo.get(Plausible.Imported.SiteImport, import_id)
# emulate oban error handler
try do
job
|> Repo.reload!()
|> ImportAnalytics.perform()
rescue
_ -> ImportAnalytics.import_fail_transient(site_import)
end
assert_receive {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}}
end)
end
end
end

Loading…
Cancel
Save