diff --git a/config/runtime.exs b/config/runtime.exs
index 90c5cce76..ee5e4cff6 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -524,8 +524,6 @@ base_queues = [
check_stats_emails: 1,
site_setup_emails: 1,
clean_invitations: 1,
- # NOTE: to be removed once #3700 is released
- google_analytics_imports: 1,
analytics_imports: 1,
domain_change_transition: 1,
check_accept_traffic_until: 1
diff --git a/lib/oban_error_reporter.ex b/lib/oban_error_reporter.ex
index 171b7d5f4..d9c3b4679 100644
--- a/lib/oban_error_reporter.ex
+++ b/lib/oban_error_reporter.ex
@@ -22,32 +22,31 @@ defmodule ObanErrorReporter do
Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
end
- # NOTE: To be cleaned up once #3700 is released
- @analytics_queues ["analytics_imports", "google_analytics_imports"]
-
defp on_job_exception(%Oban.Job{
- queue: queue,
- args: %{"site_id" => site_id, "source" => source},
+ queue: "analytics_imports",
+ args: %{"import_id" => import_id},
state: "executing",
attempt: attempt,
max_attempts: max_attempts
})
- when queue in @analytics_queues and attempt >= max_attempts do
- site = Plausible.Repo.get(Plausible.Site, site_id)
+ when attempt >= max_attempts do
+ site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
- if site do
- Plausible.Workers.ImportAnalytics.import_failed(source, site)
+ if site_import do
+ Plausible.Workers.ImportAnalytics.import_fail(site_import)
end
end
defp on_job_exception(%Oban.Job{
- queue: queue,
- args: %{"site_id" => site_id},
+ queue: "analytics_imports",
+ args: %{"import_id" => import_id},
state: "executing"
- })
- when queue in @analytics_queues do
- site = Plausible.Repo.get(Plausible.Site, site_id)
- Plausible.Purge.delete_imported_stats!(site)
+ }) do
+ site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
+
+ if site_import do
+ Plausible.Workers.ImportAnalytics.import_fail_transient(site_import)
+ end
end
defp on_job_exception(_job), do: :ignore
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
new file mode 100644
index 000000000..82623fbd2
--- /dev/null
+++ b/lib/plausible/imported.ex
@@ -0,0 +1,110 @@
+defmodule Plausible.Imported do
+ @moduledoc """
+ Context for managing site statistics imports.
+
+ Currently following importers are implemented:
+
+ * `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google
+ analytics formerly known as "Google Analytics"
+ * `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes
+ * `Plausible.Imported.CSVImporter` - a placeholder stub for CSV importer that will
+ be added soon
+
+ For more information on implementing importers, see `Plausible.Imported.Importer`.
+ """
+
+ import Ecto.Query
+
+ alias Plausible.Imported
+ alias Plausible.Imported.SiteImport
+ alias Plausible.Repo
+ alias Plausible.Site
+
+ require Plausible.Imported.SiteImport
+
+ @tables [
+ Imported.Visitor,
+ Imported.Source,
+ Imported.Page,
+ Imported.EntryPage,
+ Imported.ExitPage,
+ Imported.Location,
+ Imported.Device,
+ Imported.Browser,
+ Imported.OperatingSystem
+ ]
+
+ @table_names Enum.map(@tables, & &1.__schema__(:source))
+ # Maximum number of complete imports to account for when querying stats
+ @max_complete_imports 5
+
+ @spec tables() :: [String.t()]
+ def tables, do: @table_names
+
+ @spec load_import_data(Site.t()) :: Site.t()
+ def load_import_data(%{import_data_loaded: true} = site), do: site
+
+ def load_import_data(site) do
+ complete_import_ids = list_complete_import_ids(site)
+ earliest_import = get_earliest_import(site) || %{}
+
+ %{
+ site
+ | import_data_loaded: true,
+ earliest_import_start_date: Map.get(earliest_import, :start_date),
+ earliest_import_end_date: Map.get(earliest_import, :end_date),
+ complete_import_ids: complete_import_ids
+ }
+ end
+
+ @spec list_all_imports(Site.t()) :: [SiteImport.t()]
+ def list_all_imports(site) do
+ from(i in SiteImport, where: i.site_id == ^site.id)
+ |> Repo.all()
+ end
+
+ @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
+ def list_complete_import_ids(site) do
+ ids =
+ from(i in SiteImport,
+ where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
+ select: i.id,
+ limit: @max_complete_imports
+ )
+ |> Repo.all()
+
+ # account for legacy imports as well
+ if site.imported_data && site.imported_data.status == "ok" do
+ [0 | ids]
+ else
+ ids
+ end
+ end
+
+ @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
+ def get_earliest_import(site) do
+ first_import =
+ from(i in SiteImport,
+ where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
+ order_by: i.start_date,
+ limit: 1
+ )
+ |> Repo.one()
+
+ legacy_import =
+ if site.imported_data && site.imported_data.status == "ok" do
+ site.imported_data
+ end
+
+ [legacy_import, first_import]
+ |> Enum.reject(&is_nil/1)
+ |> Enum.min_by(& &1.start_date, Date, fn -> nil end)
+ end
+
+ @spec delete_imports_for_site(Site.t()) :: :ok
+ def delete_imports_for_site(site) do
+ Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
+
+ :ok
+ end
+end
diff --git a/lib/plausible/imported/browser.ex b/lib/plausible/imported/browser.ex
index 56d2a8772..5523bb061 100644
--- a/lib/plausible/imported/browser.ex
+++ b/lib/plausible/imported/browser.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Browser do
@primary_key false
schema "imported_browsers" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :browser, :string
field :visitors, Ch, type: "UInt64"
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
new file mode 100644
index 000000000..8c7844fe3
--- /dev/null
+++ b/lib/plausible/imported/csv_importer.ex
@@ -0,0 +1,25 @@
+defmodule Plausible.Imported.CSVImporter do
+ @moduledoc """
+ CSV importer stub.
+ """
+
+ use Plausible.Imported.Importer
+
+ @impl true
+ def name(), do: :csv
+
+ @impl true
+ def label(), do: "CSV"
+
+ # NOTE: change it once CSV import is implemented
+ @impl true
+ def email_template(), do: "google_analytics_import.html"
+
+ @impl true
+ def parse_args(%{"s3_path" => s3_path}), do: [s3_path: s3_path]
+
+ @impl true
+ def import_data(_site_import, _opts) do
+ :ok
+ end
+end
diff --git a/lib/plausible/imported/device.ex b/lib/plausible/imported/device.ex
index 896196f29..39a7d9aa4 100644
--- a/lib/plausible/imported/device.ex
+++ b/lib/plausible/imported/device.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Device do
@primary_key false
schema "imported_devices" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :device, :string
field :visitors, Ch, type: "UInt64"
diff --git a/lib/plausible/imported/entry_page.ex b/lib/plausible/imported/entry_page.ex
index 7d9b7b90d..9141bf41a 100644
--- a/lib/plausible/imported/entry_page.ex
+++ b/lib/plausible/imported/entry_page.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.EntryPage do
@primary_key false
schema "imported_entry_pages" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :entry_page, :string
field :visitors, Ch, type: "UInt64"
diff --git a/lib/plausible/imported/exit_page.ex b/lib/plausible/imported/exit_page.ex
index cbea72b84..94bd3ab7c 100644
--- a/lib/plausible/imported/exit_page.ex
+++ b/lib/plausible/imported/exit_page.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.ExitPage do
@primary_key false
schema "imported_exit_pages" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :exit_page, :string
field :visitors, Ch, type: "UInt64"
diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.ex
index 83a980ef7..bcf1eb8d4 100644
--- a/lib/plausible/imported/import_sources.ex
+++ b/lib/plausible/imported/import_sources.ex
@@ -5,11 +5,18 @@ defmodule Plausible.Imported.ImportSources do
@sources [
Plausible.Imported.UniversalAnalytics,
- Plausible.Imported.NoopImporter
+ Plausible.Imported.NoopImporter,
+ Plausible.Imported.CSVImporter
]
@sources_map Map.new(@sources, &{&1.name(), &1})
+ @source_names Enum.map(@sources, & &1.name())
+
+ @spec names() :: [atom()]
+ def names(), do: @source_names
+
+ @spec by_name(atom()) :: module()
def by_name(name) do
Map.fetch!(@sources_map, name)
end
diff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.ex
new file mode 100644
index 000000000..60cebea1c
--- /dev/null
+++ b/lib/plausible/imported/importer.ex
@@ -0,0 +1,232 @@
+defmodule Plausible.Imported.Importer do
+ @moduledoc """
+ Behaviour that should be implemented for each import source.
+
+ All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`
+ Oban worker. Each import source must define a module conforming `Importer` behaviour.
+
+ The callbacks that need to be implemented:
+
+ * `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.
+ * `label/0` - Descriptive, display friendly name of the source.
+ Example: "Google Analytics".
+ * `email_template/0` - Name of the email template to use for notifications in
+ `PlausibleWeb.Email` (`import_success` and `import_failure`). The template
+ should have content customized for a particular source.
+ * `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever
+ options were passed to `new_import/3` will be present in the input map with string
+ keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`
+ is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.
+ The expectation is parsing the map values producing a keyword list of options to
+ pass to `import_data/2`.
+ * `import_data/2` - Receives site import struct and options produced by `parse_args/1`.
+ This is where all the import processing is done. The way the import is implemented
+ is entirely arbitrary except the requirement that the process as a whole must
+ by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`
+ on successful import or `{:error, ...}` on failure. The map in success tuple is
+ used for updating site import struct and is passed to `on_success/2` callback.
+ Please note that error tuple should be only returned on errors that can't be
+ recovered from. For transient errors, the import should throw an exception or
+ simply crash.
+ * `before_start/1` - Optional callback run right before scheduling import job. It's
+ expected to either return `:ok` for the import to proceed or `{:error, ...}` tuple,
+ which will be returned from `new_import/3` call.
+ * `on_success/2` - Optional callback run once site import is completed. Receives map
+ returned from `import_data/2`. Expected to always return `:ok`.
+ * `on_failure/1` - Optional callback run when import job fails permanently.
+
+ All sources must be added to the list in `Plausible.Imported.ImportSources`.
+
+ In order to schedule a new import job using a given source, respective importer's
+ `new_import/3` function must be called. It accepts site, user who is doing the import
+ and any options necessary to carry out the import.
+
+ There's an expectation that `start_date` and `end_date` are provided either as options
+ passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters
+ are not provided, the import will eventually crash. These parameters define time range
+ of imported data which is in turn used for efficient querying.
+
+ Logic running inside `import_data/2` is expected to populated all `imported_*` tables
+ in ClickHouse with `import_id` column set to site import's ID.
+
+ Managing any configuration or authentication prior to running import is outside of
+ scope of importer logic and is expected to be implemented separately.
+
+ ## Running import fully synchronously
+
+ In case it's necessary to run the whole import job fully synchronously, the
+ `Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message
+ on completion, failure or transient failure of the import.
+
+ A basic usage scenario looks like this:
+
+ ```elixir
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(
+ site,
+ user,
+ start_date: ~D[2005-01-01],
+ end_date: Date.utc_today(),
+ # this option is necessary to setup the calling process as listener
+ listen?: true
+ )
+
+ import_id = job.args[:import_id]
+
+ receive do
+ {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}} ->
+ IO.puts("Job completed")
+
+ {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}} ->
+ IO.puts("Job failed transiently")
+
+ {:notification, :analytics_imports_jobs, %{"fail" => ^import_id}} ->
+ IO.puts("Job failed permanently")
+ after
+ 15_000 ->
+ IO.puts("Job didn't finish in 15 seconds")
+ end
+ ```
+
+ In a more realistic scenario, job scheduling will be done inside a GenServer process
+ like LiveView, where notifications can be listened for via `handle_info/2`.
+ """
+
+ alias Plausible.Imported.SiteImport
+ alias Plausible.Repo
+
+ @callback name() :: atom()
+ @callback label() :: String.t()
+ @callback email_template() :: String.t()
+ @callback parse_args(map()) :: Keyword.t()
+ @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
+ @callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
+ @callback on_success(SiteImport.t(), map()) :: :ok
+ @callback on_failure(SiteImport.t()) :: :ok
+
+ defmacro __using__(_opts) do
+ quote do
+ @behaviour Plausible.Imported.Importer
+
+ @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
+ {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
+ def new_import(site, user, opts) do
+ Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
+ end
+
+ @doc false
+ @spec run_import(SiteImport.t(), map()) :: :ok | {:error, :any}
+ def run_import(site_import, args) do
+ Plausible.Imported.Importer.run_import(
+ site_import,
+ args,
+ &parse_args/1,
+ &import_data/2,
+ &on_success/2
+ )
+ end
+
+ @doc false
+ @spec mark_failed(SiteImport.t()) :: SiteImport.t()
+ def mark_failed(site_import) do
+ site_import =
+ site_import
+ |> SiteImport.fail_changeset()
+ |> Repo.update!()
+
+ :ok = on_failure(site_import)
+
+ site_import
+ end
+
+ @impl true
+ def before_start(_site_import), do: :ok
+
+ @impl true
+ def on_success(_site_import, _extra_data), do: :ok
+
+ @impl true
+ def on_failure(_site_import), do: :ok
+
+ defoverridable before_start: 1, on_success: 2, on_failure: 1
+ end
+ end
+
+ @doc false
+ def new_import(source, site, user, opts, before_start_fun) do
+ import_params =
+ opts
+ |> Keyword.take([:start_date, :end_date])
+ |> Keyword.put(:source, source)
+ |> Map.new()
+
+ Repo.transaction(fn ->
+ result =
+ site
+ |> SiteImport.create_changeset(user, import_params)
+ |> Repo.insert()
+
+ with {:ok, site_import} <- result,
+ :ok <- before_start_fun.(site_import),
+ {:ok, job} <- schedule_job(site_import, opts) do
+ job
+ else
+ {:error, error} ->
+ Repo.rollback(error)
+ end
+ end)
+ end
+
+ @doc false
+ def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do
+ site_import =
+ site_import
+ |> SiteImport.start_changeset()
+ |> Repo.update!()
+ |> Repo.preload(:site)
+
+ import_opts = parse_args_fun.(args)
+
+ case import_fun.(site_import, import_opts) do
+ :ok ->
+ {:ok, mark_complete(site_import, %{}, on_success_fun)}
+
+ {:ok, extra_data} ->
+ {:ok, mark_complete(site_import, extra_data, on_success_fun)}
+
+ {:error, error} ->
+ {:error, error}
+ end
+ end
+
+ @oban_channel :analytics_imports_jobs
+
+ @doc false
+ def notify(site_import, event) do
+ Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})
+ end
+
+ defp schedule_job(site_import, opts) do
+ {listen?, opts} = Keyword.pop(opts, :listen?, false)
+
+ if listen? do
+ :ok = Oban.Notifier.listen([@oban_channel])
+ end
+
+ opts
+ |> Keyword.put(:import_id, site_import.id)
+ |> Map.new()
+ |> Plausible.Workers.ImportAnalytics.new()
+ |> Oban.insert()
+ end
+
+ defp mark_complete(site_import, extra_data, on_success_fun) do
+ site_import =
+ site_import
+ |> SiteImport.complete_changeset(extra_data)
+ |> Repo.update!()
+
+ :ok = on_success_fun.(site_import, extra_data)
+
+ site_import
+ end
+end
diff --git a/lib/plausible/imported/location.ex b/lib/plausible/imported/location.ex
index de48e5560..1b9e27c4f 100644
--- a/lib/plausible/imported/location.ex
+++ b/lib/plausible/imported/location.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Location do
@primary_key false
schema "imported_locations" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :country, :string
field :region, :string
diff --git a/lib/plausible/imported/noop_importer.ex b/lib/plausible/imported/noop_importer.ex
index 9f9cf0349..6dec4b078 100644
--- a/lib/plausible/imported/noop_importer.ex
+++ b/lib/plausible/imported/noop_importer.ex
@@ -3,20 +3,44 @@ defmodule Plausible.Imported.NoopImporter do
Stub import implementation.
"""
- @name "Noop"
+ use Plausible.Imported.Importer
- def name(), do: @name
+ @impl true
+ def name(), do: :noop
- def create_job(site, opts) do
- Plausible.Workers.ImportAnalytics.new(%{
- "source" => @name,
- "site_id" => site.id,
- "error" => opts[:error]
- })
- end
+ @impl true
+ def label(), do: "Noop"
+
+ # reusing existing template from another source
+ @impl true
+ def email_template(), do: "google_analytics_import.html"
+ @impl true
def parse_args(opts), do: opts
- def import(_site, %{"error" => true}), do: {:error, "Something went wrong"}
- def import(_site, _opts), do: :ok
+ @impl true
+ def import_data(_site_import, %{"error" => true}), do: {:error, "Something went wrong"}
+ def import_data(_site_import, %{"crash" => true}), do: raise("boom")
+ def import_data(_site_import, _opts), do: :ok
+
+ @impl true
+ def before_start(site_import) do
+ send(self(), {:before_start, site_import.id})
+
+ :ok
+ end
+
+ @impl true
+ def on_success(site_import, _extra_data) do
+ send(self(), {:on_success, site_import.id})
+
+ :ok
+ end
+
+ @impl true
+ def on_failure(site_import) do
+ send(self(), {:on_failure, site_import.id})
+
+ :ok
+ end
end
diff --git a/lib/plausible/imported/operating_system.ex b/lib/plausible/imported/operating_system.ex
index fc5292ddb..3069ac39b 100644
--- a/lib/plausible/imported/operating_system.ex
+++ b/lib/plausible/imported/operating_system.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.OperatingSystem do
@primary_key false
schema "imported_operating_systems" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :operating_system, :string
field :visitors, Ch, type: "UInt64"
diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.ex
index 7a320ba24..390463e06 100644
--- a/lib/plausible/imported/page.ex
+++ b/lib/plausible/imported/page.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Page do
@primary_key false
schema "imported_pages" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :hostname, :string
field :page, :string
diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex
deleted file mode 100644
index e304f1e83..000000000
--- a/lib/plausible/imported/site.ex
+++ /dev/null
@@ -1,13 +0,0 @@
-defmodule Plausible.Imported do
- @tables ~w(
- imported_visitors imported_sources imported_pages imported_entry_pages
- imported_exit_pages imported_locations imported_devices imported_browsers
- imported_operating_systems
- )
- @spec tables() :: [String.t()]
- def tables, do: @tables
-
- def forget(site) do
- Plausible.Purge.delete_imported_stats!(site)
- end
-end
diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.ex
new file mode 100644
index 000000000..d8774fcfb
--- /dev/null
+++ b/lib/plausible/imported/site_import.ex
@@ -0,0 +1,68 @@
+defmodule Plausible.Imported.SiteImport do
+ @moduledoc """
+ Site import schema.
+ """
+
+ use Ecto.Schema
+
+ import Ecto.Changeset
+
+ alias Plausible.Auth.User
+ alias Plausible.Imported.ImportSources
+ alias Plausible.Site
+
+ @statuses [:pending, :importing, :completed, :failed]
+
+ @type t() :: %__MODULE__{}
+
+ schema "site_imports" do
+ field :start_date, :date
+ field :end_date, :date
+ field :source, Ecto.Enum, values: ImportSources.names()
+ field :status, Ecto.Enum, values: @statuses
+
+ belongs_to :site, Site
+ belongs_to :imported_by, User
+
+ timestamps()
+ end
+
+ for status <- @statuses do
+ defmacro unquote(status)(), do: unquote(status)
+ end
+
+ # NOTE: this is necessary for backwards compatibility
+ # with legacy imports
+ @spec label(t() | Site.ImportedData.t()) :: String.t()
+ def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()
+ def label(%Site.ImportedData{source: source}), do: source
+
+ @spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
+ def create_changeset(site, user, params) do
+ %__MODULE__{}
+ |> cast(params, [:source, :start_date, :end_date])
+ |> validate_required([:source])
+ |> put_assoc(:site, site)
+ |> put_assoc(:imported_by, user)
+ |> put_change(:status, pending())
+ end
+
+ @spec start_changeset(t()) :: Ecto.Changeset.t()
+ def start_changeset(site_import) do
+ site_import
+ |> change(status: importing())
+ end
+
+ @spec complete_changeset(t(), map()) :: Ecto.Changeset.t()
+ def complete_changeset(site_import, params \\ %{}) do
+ site_import
+ |> cast(params, [:start_date, :end_date])
+ |> put_change(:status, completed())
+ |> validate_required([:start_date, :end_date])
+ end
+
+ @spec fail_changeset(t()) :: Ecto.Changeset.t()
+ def fail_changeset(site_import) do
+ change(site_import, status: failed())
+ end
+end
diff --git a/lib/plausible/imported/source.ex b/lib/plausible/imported/source.ex
index c19ca07c6..3ea5cc619 100644
--- a/lib/plausible/imported/source.ex
+++ b/lib/plausible/imported/source.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Source do
@primary_key false
schema "imported_sources" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :source, :string
field :utm_medium, :string
diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex
index 243704a7e..246be89f4 100644
--- a/lib/plausible/imported/universal_analytics.ex
+++ b/lib/plausible/imported/universal_analytics.ex
@@ -3,50 +3,59 @@ defmodule Plausible.Imported.UniversalAnalytics do
Import implementation for Universal Analytics.
"""
- use Plausible.ClickhouseRepo
+ use Plausible.Imported.Importer
- alias Plausible.Site
+ alias Plausible.Repo
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
- @type job_opt() ::
- {:view_id, non_neg_integer()}
- | {:start_date | :end_date | :access_token | :refresh_token | :token_expires_at,
- String.t()}
+ @impl true
+ def name(), do: :universal_analytics
- @type import_opt() ::
- {:view_id, non_neg_integer()}
- | {:date_range, Date.Range.t()}
- | {:auth, {String.t(), String.t(), String.t()}}
+ @impl true
+ def label(), do: "Google Analytics"
- # NOTE: we have to use old name for now
- @name "Google Analytics"
+ @impl true
+ def email_template(), do: "google_analytics_import.html"
- @spec name() :: String.t()
- def name(), do: @name
+ @impl true
+ def before_start(site_import) do
+ site = Repo.preload(site_import, :site).site
- @spec create_job(Site.t(), [job_opt()]) :: Ecto.Changeset.t()
- def create_job(site, opts) do
- view_id = Keyword.fetch!(opts, :view_id)
- start_date = Keyword.fetch!(opts, :start_date)
- end_date = Keyword.fetch!(opts, :end_date)
- access_token = Keyword.fetch!(opts, :access_token)
- refresh_token = Keyword.fetch!(opts, :refresh_token)
- token_expires_at = Keyword.fetch!(opts, :token_expires_at)
-
- Plausible.Workers.ImportAnalytics.new(%{
- "source" => @name,
- "site_id" => site.id,
- "view_id" => view_id,
- "start_date" => start_date,
- "end_date" => end_date,
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "token_expires_at" => token_expires_at
- })
+ site
+ |> Plausible.Site.start_import(
+ site_import.start_date,
+ site_import.end_date,
+ label()
+ )
+ |> Repo.update!()
+
+ :ok
+ end
+
+ @impl true
+ def on_success(site_import, _extra_data) do
+ site = Repo.preload(site_import, :site).site
+
+ site
+ |> Plausible.Site.import_success()
+ |> Repo.update!()
+
+ :ok
+ end
+
+ @impl true
+ def on_failure(site_import) do
+ site = Repo.preload(site_import, :site).site
+
+ site
+ |> Plausible.Site.import_failure()
+ |> Repo.update!()
+
+ :ok
end
- @spec parse_args(map()) :: [import_opt()]
+ @impl true
def parse_args(
%{"view_id" => view_id, "start_date" => start_date, "end_date" => end_date} = args
) do
@@ -73,8 +82,8 @@ defmodule Plausible.Imported.UniversalAnalytics do
This function fetches Google Analytics reports which are then passed in batches
to Clickhouse by the `Plausible.Imported.Buffer` process.
"""
- @spec import(Site.t(), [import_opt()]) :: :ok | {:error, any()}
- def import(site, opts) do
+ @impl true
+ def import_data(site_import, opts) do
date_range = Keyword.fetch!(opts, :date_range)
view_id = Keyword.fetch!(opts, :view_id)
auth = Keyword.fetch!(opts, :auth)
@@ -82,7 +91,7 @@ defmodule Plausible.Imported.UniversalAnalytics do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
- records = from_report(rows, site.id, table)
+ records = from_report(rows, site_import.site_id, site_import.id, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
@@ -94,14 +103,14 @@ defmodule Plausible.Imported.UniversalAnalytics do
end
end
- def from_report(nil, _site_id, _metric), do: nil
+ def from_report(nil, _site_id, _import_id, _metric), do: nil
- def from_report(data, site_id, table) do
+ def from_report(data, site_id, import_id, table) do
Enum.reduce(data, [], fn row, acc ->
if Map.get(row.dimensions, "ga:date") in @missing_values do
acc
else
- [new_from_report(site_id, table, row) | acc]
+ [new_from_report(site_id, import_id, table, row) | acc]
end
end)
end
@@ -111,9 +120,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
round(float)
end
- defp new_from_report(site_id, "imported_visitors", row) do
+ defp new_from_report(site_id, import_id, "imported_visitors", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("ga:pageviews") |> parse_number(),
@@ -123,9 +133,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_sources", row) do
+ defp new_from_report(site_id, import_id, "imported_sources", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("ga:source") |> parse_referrer(),
utm_medium: row.dimensions |> Map.fetch!("ga:medium") |> default_if_missing(),
@@ -139,9 +150,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_pages", row) do
+ defp new_from_report(site_id, import_id, "imported_pages", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
hostname: row.dimensions |> Map.fetch!("ga:hostname") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("ga:pagePath") |> URI.parse() |> Map.get(:path),
@@ -152,9 +164,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_entry_pages", row) do
+ defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
entry_page: row.dimensions |> Map.fetch!("ga:landingPagePath"),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@@ -164,9 +177,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_exit_pages", row) do
+ defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
exit_page: Map.fetch!(row.dimensions, "ga:exitPagePath"),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@@ -174,13 +188,14 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_locations", row) do
+ defp new_from_report(site_id, import_id, "imported_locations", row) do
country_code = row.dimensions |> Map.fetch!("ga:countryIsoCode") |> default_if_missing("")
city_name = row.dimensions |> Map.fetch!("ga:city") |> default_if_missing("")
city_data = Location.get_city(city_name, country_code)
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
country: country_code,
region: row.dimensions |> Map.fetch!("ga:regionIsoCode") |> default_if_missing(""),
@@ -192,9 +207,10 @@ defmodule Plausible.Imported.UniversalAnalytics do
}
end
- defp new_from_report(site_id, "imported_devices", row) do
+ defp new_from_report(site_id, import_id, "imported_devices", row) do
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
device: row.dimensions |> Map.fetch!("ga:deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@@ -214,11 +230,12 @@ defmodule Plausible.Imported.UniversalAnalytics do
"(not set)" => ""
}
- defp new_from_report(site_id, "imported_browsers", row) do
+ defp new_from_report(site_id, import_id, "imported_browsers", row) do
browser = Map.fetch!(row.dimensions, "ga:browser")
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
@@ -234,11 +251,12 @@ defmodule Plausible.Imported.UniversalAnalytics do
"(not set)" => ""
}
- defp new_from_report(site_id, "imported_operating_systems", row) do
+ defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
os = Map.fetch!(row.dimensions, "ga:operatingSystem")
%{
site_id: site_id,
+ import_id: import_id,
date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os),
visitors: row.metrics |> Map.fetch!("ga:users") |> parse_number(),
diff --git a/lib/plausible/imported/visitor.ex b/lib/plausible/imported/visitor.ex
index 7cad7a196..2e4d4d26d 100644
--- a/lib/plausible/imported/visitor.ex
+++ b/lib/plausible/imported/visitor.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.Visitor do
@primary_key false
schema "imported_visitors" do
field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
field :date, :date
field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index b774798ba..9fe11b077 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -33,18 +33,42 @@ defmodule Plausible.Purge do
- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)
"""
- @spec delete_imported_stats!(Plausible.Site.t()) :: :ok
+ alias Plausible.Repo
+
+ @spec delete_imported_stats!(Plausible.Site.t() | Plausible.Imported.SiteImport.t()) :: :ok
@doc """
- Deletes imported stats from Google Analytics, and clears the
- `stats_start_date` field.
+ Deletes imported stats from and clears the `stats_start_date` field.
+
+ The `stats_start_date` is expected to get repopulated the next time
+ `Plausible.Sites.stats_start_date/1` is called.
+
+ If the input argument is a site, all imported stats are deleted. If it's a site import,
+ only imported stats for that import are deleted.
"""
- def delete_imported_stats!(site) do
+ def delete_imported_stats!(%Plausible.Site{} = site) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])
end)
- clear_stats_start_date!(site)
+ Plausible.Sites.clear_stats_start_date!(site)
+
+ :ok
+ end
+
+ def delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) do
+ site_import = Repo.preload(site_import, :site)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"
+
+ Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [
+ site_import.site_id,
+ site_import.id
+ ])
+ end)
+
+ Plausible.Sites.clear_stats_start_date!(site_import.site)
:ok
end
@@ -67,10 +91,4 @@ defmodule Plausible.Purge do
)
|> Plausible.Repo.update!()
end
-
- defp clear_stats_start_date!(site) do
- site
- |> Ecto.Changeset.change(stats_start_date: nil)
- |> Plausible.Repo.update!()
- end
end
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index df99de2f6..c5124f6e2 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -6,7 +6,6 @@ defmodule Plausible.Site do
import Ecto.Changeset
alias Plausible.Auth.User
alias Plausible.Site.GoogleAuth
- alias Plausible.Timezones
@type t() :: %__MODULE__{}
@@ -55,6 +54,14 @@ defmodule Plausible.Site do
field :entry_type, :string, virtual: true
field :pinned_at, :naive_datetime, virtual: true
+ # Used for caching imports data for the duration of the whole request
+ # to avoid multiple identical fetches. Populated by plugs putting
+ # `site` in `assigns`.
+ field :import_data_loaded, :boolean, default: false, virtual: true
+ field :earliest_import_start_date, :date, virtual: true
+ field :earliest_import_end_date, :date, virtual: true
+ field :complete_import_ids, {:array, :integer}, default: [], virtual: true
+
timestamps()
end
@@ -159,7 +166,6 @@ defmodule Plausible.Site do
def import_success(site) do
change(site,
- stats_start_date: site.imported_data.start_date,
imported_data: %{status: "ok"}
)
end
@@ -168,50 +174,10 @@ defmodule Plausible.Site do
change(site, imported_data: %{status: "error"})
end
- def set_imported_source(site, imported_source) do
- change(site,
- imported_data: %Plausible.Site.ImportedData{
- end_date: Timex.today(),
- source: imported_source
- }
- )
- end
-
def remove_imported_data(site) do
change(site, imported_data: nil)
end
- @doc """
- Returns the date of the first recorded stat in the timezone configured by the user.
- This function does 2 transformations:
- UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date
-
- ## Examples
-
- iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})
- nil
-
- iex> utc_start = ~N[2022-09-28 00:00:00]
- iex> tz = "Europe/Helsinki"
- iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
- iex> Plausible.Site.local_start_date(site)
- ~D[2022-09-28]
-
- iex> utc_start = ~N[2022-09-28 00:00:00]
- iex> tz = "America/Los_Angeles"
- iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
- iex> Plausible.Site.local_start_date(site)
- ~D[2022-09-27]
- """
- def local_start_date(%__MODULE__{stats_start_date: nil}) do
- nil
- end
-
- def local_start_date(site) do
- site.stats_start_date
- |> Timezones.to_date_in_timezone(site.timezone)
- end
-
defp clean_domain(changeset) do
clean_domain =
(get_field(changeset, :domain) || "")
diff --git a/lib/plausible/site/imported_data.ex b/lib/plausible/site/imported_data.ex
index c1db36cae..bec7e571e 100644
--- a/lib/plausible/site/imported_data.ex
+++ b/lib/plausible/site/imported_data.ex
@@ -4,6 +4,8 @@ defmodule Plausible.Site.ImportedData do
"""
use Ecto.Schema
+ @type t() :: %__MODULE__{}
+
embedded_schema do
field :start_date, :date
field :end_date, :date
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 14fc2f0f5..1c8ac51e1 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -189,13 +189,50 @@ defmodule Plausible.Sites do
end
end
+ @spec clear_stats_start_date!(Site.t()) :: Site.t()
+ def clear_stats_start_date!(site) do
+ site
+ |> Ecto.Changeset.change(stats_start_date: nil)
+ |> Plausible.Repo.update!()
+ end
+
+ @doc """
+ Returns the date of the first recorded stat in the timezone configured by the user.
+ This function does 2 transformations:
+ UTC %NaiveDateTime{} -> Local %DateTime{} -> Local %Date
+
+ ## Examples
+
+ iex> Plausible.Site.local_start_date(%Plausible.Site{stats_start_date: nil})
+ nil
+
+ iex> utc_start = ~N[2022-09-28 00:00:00]
+ iex> tz = "Europe/Helsinki"
+ iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
+ iex> Plausible.Site.local_start_date(site)
+ ~D[2022-09-28]
+
+ iex> utc_start = ~N[2022-09-28 00:00:00]
+ iex> tz = "America/Los_Angeles"
+ iex> site = %Plausible.Site{stats_start_date: utc_start, timezone: tz}
+ iex> Plausible.Site.local_start_date(site)
+ ~D[2022-09-27]
+ """
+ @spec local_start_date(Site.t()) :: Date.t() | nil
+ def local_start_date(site) do
+ if stats_start_date = stats_start_date(site) do
+ Plausible.Timezones.to_date_in_timezone(stats_start_date, site.timezone)
+ end
+ end
+
@spec stats_start_date(Site.t()) :: Date.t() | nil
@doc """
Returns the date of the first event of the given site, or `nil` if the site
does not have stats yet.
If this is the first time the function is called for the site, it queries
- Clickhouse and saves the date in the sites table.
+ imported stats and Clickhouse, choosing the earliest start date and saves
+ it in the sites table.
"""
def stats_start_date(site)
@@ -204,7 +241,17 @@ defmodule Plausible.Sites do
end
def stats_start_date(%Site{} = site) do
- if start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(site) do
+ site = Plausible.Imported.load_import_data(site)
+
+ start_date =
+ [
+ site.earliest_import_start_date,
+ Plausible.Stats.Clickhouse.pageview_start_date_local(site)
+ ]
+ |> Enum.reject(&is_nil/1)
+ |> Enum.min(Date, fn -> nil end)
+
+ if start_date do
updated_site =
site
|> Site.set_stats_start_date(start_date)
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 394ba6fe8..8cfe02384 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -16,9 +16,12 @@ defmodule Plausible.Stats.Imported do
query,
metrics
) do
+ import_ids = site.complete_import_ids
+
imported_q =
from(v in "imported_visitors",
where: v.site_id == ^site.id,
+ where: v.import_id in ^import_ids,
where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last,
select: %{}
)
@@ -107,11 +110,14 @@ defmodule Plausible.Stats.Imported do
{"imported_#{dim}s", String.to_existing_atom(dim)}
end
+ import_ids = site.complete_import_ids
+
imported_q =
from(
i in table,
group_by: field(i, ^dim),
where: i.site_id == ^site.id,
+ where: i.import_id in ^import_ids,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.visitors > 0,
select: %{}
@@ -315,10 +321,13 @@ defmodule Plausible.Stats.Imported do
end
def merge_imported(q, site, query, :aggregate, metrics) do
+ import_ids = site.complete_import_ids
+
imported_q =
from(
i in "imported_visitors",
where: i.site_id == ^site.id,
+ where: i.import_id in ^import_ids,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
select: %{}
)
diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex
index 4ad13a9c3..116f711c9 100644
--- a/lib/plausible/stats/interval.ex
+++ b/lib/plausible/stats/interval.ex
@@ -78,7 +78,7 @@ defmodule Plausible.Stats.Interval do
@valid_by_period
end
- with %Date{} = stats_start <- site.stats_start_date,
+ with %Date{} = stats_start <- Plausible.Sites.stats_start_date(site),
true <- abs(Timex.diff(Date.utc_today(), stats_start, :months)) > 12 do
Map.replace(table, "all", ["week", "month"])
else
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 9432ccfd4..934c293dd 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -121,7 +121,7 @@ defmodule Plausible.Stats.Query do
defp put_period(query, site, %{"period" => "all"}) do
now = today(site.timezone)
- start_date = Plausible.Site.local_start_date(site) || now
+ start_date = Plausible.Sites.local_start_date(site) || now
struct!(query,
period: "all",
@@ -251,9 +251,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do
cond do
- is_nil(site.imported_data) -> false
- site.imported_data.status != "ok" -> false
- Timex.after?(query.date_range.first, site.imported_data.end_date) -> false
+ is_nil(site.earliest_import_end_date) -> false
+ Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
Enum.any?(query.filters) -> false
query.period == "realtime" -> false
true -> requested?
diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.ex
index def1a3580..03461e35e 100644
--- a/lib/plausible/timezones.ex
+++ b/lib/plausible/timezones.ex
@@ -22,7 +22,14 @@ defmodule Plausible.Timezones do
@spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t()
def to_date_in_timezone(dt, timezone) do
- utc_dt = Timex.Timezone.convert(dt, "UTC")
+ utc_dt =
+ case dt do
+ %Date{} ->
+ Timex.to_datetime(dt, "UTC")
+
+ dt ->
+ Timex.Timezone.convert(dt, "UTC")
+ end
case Timex.Timezone.convert(utc_dt, timezone) do
%DateTime{} = tz_dt ->
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index dda5eb3a0..97f9692ef 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -3,6 +3,8 @@ defmodule PlausibleWeb.Api.StatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
+
+ alias Plausible.Imported.SiteImport
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
@@ -132,6 +134,8 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels)
+ site_import = Plausible.Imported.get_earliest_import(site)
+
json(conn, %{
plot: plot_timeseries(timeseries_result, selected_metric),
labels: labels,
@@ -140,7 +144,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: with_imported?(query, comparison_query),
- imported_source: site.imported_data && site.imported_data.source,
+ imported_source: site_import && SiteImport.label(site_import),
full_intervals: full_intervals
})
else
@@ -216,12 +220,14 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
+ site_import = Plausible.Imported.get_earliest_import(site)
+
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query),
- imported_source: site.imported_data && site.imported_data.source,
+ imported_source: site_import && SiteImport.label(site_import),
comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first,
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index aa2775b0b..e34581302 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -764,11 +764,12 @@ defmodule PlausibleWeb.SiteController do
"expires_at" => expires_at
}) do
site = conn.assigns.site
- source = "Google Analytics"
+ current_user = conn.assigns.current_user
- job =
- Plausible.Imported.UniversalAnalytics.create_job(
+ {:ok, _} =
+ Plausible.Imported.UniversalAnalytics.new_import(
site,
+ current_user,
view_id: view_id,
start_date: start_date,
end_date: end_date,
@@ -777,49 +778,46 @@ defmodule PlausibleWeb.SiteController do
token_expires_at: expires_at
)
- Ecto.Multi.new()
- |> Ecto.Multi.update(
- :update_site,
- Plausible.Site.start_import(site, start_date, end_date, source)
- )
- |> Oban.insert(:oban_job, job)
- |> Repo.transaction()
-
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
- # NOTE: To be cleaned up once #3700 is released
- @analytics_queues ["analytics_imports", "google_analytics_imports"]
-
def forget_imported(conn, _params) do
- site = conn.assigns[:site]
+ site = conn.assigns.site
- cond do
- site.imported_data ->
- Oban.cancel_all_jobs(
- from j in Oban.Job,
- where:
- j.queue in @analytics_queues and
- fragment("(? ->> 'site_id')::int", j.args) == ^site.id
- )
+ import_ids =
+ site
+ |> Plausible.Imported.list_all_imports()
+ |> Enum.map(& &1.id)
+
+ import_ids =
+ if site.imported_data do
+ [0 | import_ids]
+ else
+ import_ids
+ end
- Plausible.Imported.forget(site)
+ if import_ids != [] do
+ Oban.cancel_all_jobs(
+ from j in Oban.Job,
+ where:
+ j.queue == "analytics_imports" and
+ fragment("(? ->> 'import_id')::int", j.args) in ^import_ids
+ )
- site
- |> Plausible.Site.remove_imported_data()
- |> Repo.update!()
+ Plausible.Purge.delete_imported_stats!(site)
- conn
- |> put_flash(:success, "Imported data has been cleared")
- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
+ Plausible.Imported.delete_imports_for_site(site)
- true ->
- conn
- |> put_flash(:error, "No data has been imported")
- |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
+ site
+ |> Plausible.Site.remove_imported_data()
+ |> Repo.update!()
end
+
+ conn
+ |> put_flash(:success, "Imported data has been cleared")
+ |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
def change_domain(conn, _params) do
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 55910dc3b..422b7d78d 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -309,6 +309,7 @@ defmodule PlausibleWeb.StatsController do
cond do
!shared_link.site.locked ->
shared_link = Plausible.Repo.preload(shared_link, site: :owner)
+ stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)
conn
|> put_resp_header("x-robots-tag", "noindex, nofollow")
@@ -318,7 +319,7 @@ defmodule PlausibleWeb.StatsController do
has_goals: Sites.has_goals?(shared_link.site),
funnels: list_funnels(shared_link.site),
has_props: Plausible.Props.configured?(shared_link.site),
- stats_start_date: shared_link.site.stats_start_date,
+ stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: title(conn, shared_link.site),
offer_email_report: false,
diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex
index 427f031f1..76af84627 100644
--- a/lib/plausible_web/email.ex
+++ b/lib/plausible_web/email.ex
@@ -313,29 +313,35 @@ defmodule PlausibleWeb.Email do
)
end
- # NOTE: make email different depending on import source
- def import_success(source, user, site) do
+ def import_success(site_import, user) do
+ import_api = Plausible.Imported.ImportSources.by_name(site_import.source)
+ label = import_api.label()
+
priority_email()
|> to(user)
|> tag("import-success-email")
- |> subject("#{source} data imported for #{site.domain}")
- |> render("google_analytics_import.html", %{
- site: site,
- link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain),
+ |> subject("#{label} data imported for #{site_import.site.domain}")
+ |> render(import_api.email_template(), %{
+ site_import: site_import,
+ label: label,
+ link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site_import.site.domain),
user: user,
success: true
})
end
- # NOTE: make email different depending on import source
- def import_failure(source, user, site) do
+ def import_failure(site_import, user) do
+ import_api = Plausible.Imported.ImportSources.by_name(site_import.source)
+ label = import_api.label()
+
priority_email()
|> to(user)
|> tag("import-failure-email")
- |> subject("#{source} import failed for #{site.domain}")
- |> render("google_analytics_import.html", %{
+ |> subject("#{label} import failed for #{site_import.site.domain}")
+ |> render(import_api.email_template(), %{
+ site_import: site_import,
+ label: label,
user: user,
- site: site,
success: false
})
end
diff --git a/lib/plausible_web/plugs/authorize_site_access.ex b/lib/plausible_web/plugs/authorize_site_access.ex
index 16995a5bc..de35461c5 100644
--- a/lib/plausible_web/plugs/authorize_site_access.ex
+++ b/lib/plausible_web/plugs/authorize_site_access.ex
@@ -47,6 +47,8 @@ defmodule PlausibleWeb.AuthorizeSiteAccess do
Sentry.Context.set_extra_context(%{site_id: site.id, domain: site.domain})
Plausible.OpenTelemetry.add_site_attributes(site)
+ site = Plausible.Imported.load_import_data(site)
+
merge_assigns(conn, site: site, current_user_role: role)
else
PlausibleWeb.ControllerHelpers.render_error(conn, 404) |> halt
diff --git a/lib/plausible_web/plugs/authorize_stats_api.ex b/lib/plausible_web/plugs/authorize_stats_api.ex
index b45c41de9..4dc68b5bd 100644
--- a/lib/plausible_web/plugs/authorize_stats_api.ex
+++ b/lib/plausible_web/plugs/authorize_stats_api.ex
@@ -16,6 +16,7 @@ defmodule PlausibleWeb.AuthorizeStatsApiPlug do
:ok <- check_api_key_rate_limit(api_key),
{:ok, site} <- verify_access(api_key, conn.params["site_id"]) do
Plausible.OpenTelemetry.add_site_attributes(site)
+ site = Plausible.Imported.load_import_data(site)
assign(conn, :site, site)
else
{:error, :missing_api_key} ->
diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.heex b/lib/plausible_web/templates/email/google_analytics_import.html.heex
index d7d1ce539..a9879165c 100644
--- a/lib/plausible_web/templates/email/google_analytics_import.html.heex
+++ b/lib/plausible_web/templates/email/google_analytics_import.html.heex
@@ -1,11 +1,11 @@
<%= if @success do %>
- Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site.domain %> now contains historical imported data from <%= date_format(
- @site.imported_data.start_date
- ) %> to <%= date_format(@site.imported_data.end_date) %>
+ Your Google Analytics import has completed successfully. The Plausible dashboard for <%= @site_import.site.domain %> now contains historical imported data from <%= date_format(
+ @site_import.start_date
+ ) %> to <%= date_format(@site_import.end_date) %>
<%= link("Click here", to: @link) %> to view your dashboard.
<% else %>
- Unfortunately, your Google Analytics import for <%= @site.domain %> did not complete successfully. Sorry about that!
+ Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
diff --git a/lib/workers/import_analytics.ex b/lib/workers/import_analytics.ex
index 0be5885bb..7675419b3 100644
--- a/lib/workers/import_analytics.ex
+++ b/lib/workers/import_analytics.ex
@@ -9,33 +9,41 @@ defmodule Plausible.Workers.ImportAnalytics do
use Oban.Worker,
queue: :analytics_imports,
max_attempts: 3,
- unique: [fields: [:args], period: 60]
+ unique: [fields: [:args], keys: [:import_id], period: 60]
alias Plausible.Imported.ImportSources
+ alias Plausible.Imported.Importer
+ alias Plausible.Imported.SiteImport
@impl Oban.Worker
def perform(%Oban.Job{
- args: %{"site_id" => site_id, "source" => source} = args
+ args: %{"import_id" => import_id} = args
}) do
- import_api = ImportSources.by_name(source)
- import_opts = import_api.parse_args(args)
+ site_import =
+ SiteImport
+ |> Repo.get!(import_id)
+ |> Repo.preload(:site)
- site = Repo.get!(Plausible.Site, site_id)
+ import_api = ImportSources.by_name(site_import.source)
- case import_api.import(site, import_opts) do
- :ok ->
- import_success(source, site)
+ case import_api.run_import(site_import, args) do
+ {:ok, site_import} ->
+ import_complete(site_import)
:ok
{:error, error} ->
- Sentry.capture_message("Failed to import from Google Analytics",
- extra: %{site: site.domain, error: inspect(error)}
+ Sentry.capture_message("Failed to import from #{site_import.source}",
+ extra: %{
+ import_id: site_import.id,
+ site: site_import.site.domain,
+ error: inspect(error)
+ }
)
- import_failed(source, site)
+ import_fail(site_import)
- {:error, error}
+ {:discard, error}
end
end
@@ -45,33 +53,44 @@ defmodule Plausible.Workers.ImportAnalytics do
300
end
- def import_success(source, site) do
- site = Repo.preload(site, memberships: :user)
+ def import_complete(site_import) do
+ site_import = Repo.preload(site_import, site: [memberships: :user])
- site
- |> Plausible.Site.import_success()
- |> Repo.update!()
-
- Enum.each(site.memberships, fn membership ->
+ Enum.each(site_import.site.memberships, fn membership ->
if membership.role in [:owner, :admin] do
- PlausibleWeb.Email.import_success(source, membership.user, site)
+ PlausibleWeb.Email.import_success(site_import, membership.user)
|> Plausible.Mailer.send()
end
end)
+
+ Plausible.Sites.clear_stats_start_date!(site_import.site)
+
+ Importer.notify(site_import, :complete)
+
+ :ok
end
- def import_failed(source, site) do
- site = Repo.preload(site, memberships: :user)
+ def import_fail_transient(site_import) do
+ Plausible.Purge.delete_imported_stats!(site_import)
+
+ Importer.notify(site_import, :transient_fail)
+ end
+
+ def import_fail(site_import) do
+ Plausible.Purge.delete_imported_stats!(site_import)
+
+ import_api = ImportSources.by_name(site_import.source)
- site
- |> Plausible.Site.import_failure()
- |> Repo.update!()
+ site_import =
+ site_import
+ |> import_api.mark_failed()
+ |> Repo.preload(site: [memberships: :user])
- Plausible.Purge.delete_imported_stats!(site)
+ Importer.notify(site_import, :fail)
- Enum.each(site.memberships, fn membership ->
+ Enum.each(site_import.site.memberships, fn membership ->
if membership.role in [:owner, :admin] do
- PlausibleWeb.Email.import_failure(source, membership.user, site)
+ PlausibleWeb.Email.import_failure(site_import, membership.user)
|> Plausible.Mailer.send()
end
end)
diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs
index 92897270a..c448e7c43 100644
--- a/test/plausible/google/api_test.exs
+++ b/test/plausible/google/api_test.exs
@@ -40,7 +40,7 @@ defmodule Plausible.Google.ApiTest do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
- records = UniversalAnalytics.from_report(rows, site.id, table)
+ records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
@@ -82,7 +82,7 @@ defmodule Plausible.Google.ApiTest do
{:ok, buffer} = Plausible.Imported.Buffer.start_link()
persist_fn = fn table, rows ->
- records = UniversalAnalytics.from_report(rows, site.id, table)
+ records = UniversalAnalytics.from_report(rows, site.id, _import_id = 123, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs
index dead7ba12..03ccfb0c0 100644
--- a/test/plausible/imported/imported_test.exs
+++ b/test/plausible/imported/imported_test.exs
@@ -4,1109 +4,1219 @@ defmodule Plausible.ImportedTest do
@user_id 123
- defp import_data(ga_data, site_id, table_name) do
+ defp import_data(ga_data, site_id, import_id, table_name) do
ga_data
- |> Plausible.Imported.UniversalAnalytics.from_report(site_id, table_name)
+ |> Plausible.Imported.UniversalAnalytics.from_report(site_id, import_id, table_name)
|> then(&Plausible.Imported.Buffer.insert_all(table_name, &1))
end
- describe "Parse and import third party data fetched from Google Analytics" do
- setup [:create_user, :log_in, :create_new_site, :add_imported_data]
-
- test "Visitors data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
- }
- },
- %{
- dimensions: %{"ga:date" => "20210131"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
+ for import_type <- [:legacy, :new_and_legacy, :new] do
+ describe "Parse and import third party data fetched from Google Analytics as #{import_type} import" do
+ if import_type == :new do
+ setup [:create_user, :log_in, :create_new_site]
+ else
+ setup [:create_user, :log_in, :create_new_site, :add_imported_data]
+ end
+
+ setup %{user: user, site: site} do
+ if unquote(import_type) in [:new, :new_and_legacy] do
+ import_params =
+ if unquote(import_type) == :new_and_legacy do
+ site.imported_data
+ |> Map.from_struct()
+ |> Map.put(:source, :universal_analytics)
+ else
+ %{
+ source: :universal_analytics,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today()
+ }
+ end
+
+ site_import =
+ site
+ |> Plausible.Imported.SiteImport.create_changeset(
+ user,
+ import_params
+ )
+ |> Plausible.Repo.insert!()
+ |> Plausible.Imported.SiteImport.complete_changeset()
+ |> Plausible.Repo.update!()
+
+ {:ok, %{import_id: site_import.id}}
+ else
+ {:ok, %{import_id: 0}}
+ end
+ end
+
+ test "Visitors data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210131"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
}
- }
- ],
- site.id,
- "imported_visitors"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_visitors"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
-
- assert Enum.count(plot) == 31
- assert List.first(plot) == 2
- assert List.last(plot) == 2
- assert Enum.sum(plot) == 4
- end
-
- test "returns data grouped by week", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
- }
- },
- %{
- dimensions: %{"ga:date" => "20210131"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
+ )
+
+ assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
+ json_response(conn, 200)
+
+ assert Enum.count(plot) == 31
+ assert List.first(plot) == 2
+ assert List.last(plot) == 2
+ assert Enum.sum(plot) == 4
+ end
+
+ test "returns data grouped by week", %{conn: conn, site: site, import_id: import_id} do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210131"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
}
- }
- ],
- site.id,
- "imported_visitors"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
+ ],
+ site.id,
+ import_id,
+ "imported_visitors"
)
- assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200)
- assert Enum.count(plot) == 5
- assert List.first(plot) == 2
- assert List.last(plot) == 2
- assert Enum.sum(plot) == 4
- end
-
- test "Sources are imported", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- referrer_source: "Google",
- referrer: "google.com",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- referrer_source: "Google",
- referrer: "google.com",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- referrer_source: "DuckDuckGo",
- referrer: "duckduckgo.com",
- timestamp: ~N[2021-01-01 00:00:00]
- )
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "organic",
- "ga:source" => "duckduckgo.com"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
+ )
+
+ assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
+ json_response(conn, 200)
+
+ assert Enum.count(plot) == 5
+ assert List.first(plot) == 2
+ assert List.last(plot) == 2
+ assert Enum.sum(plot) == 4
+ end
+
+ test "Sources are imported", %{conn: conn, site: site, import_id: import_id} do
+ populate_stats(site, [
+ build(:pageview,
+ referrer_source: "Google",
+ referrer: "google.com",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ referrer_source: "Google",
+ referrer: "google.com",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ referrer_source: "DuckDuckGo",
+ referrer: "duckduckgo.com",
+ timestamp: ~N[2021-01-01 00:00:00]
+ )
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "organic",
+ "ga:source" => "duckduckgo.com"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210131",
- "ga:keyword" => "",
- "ga:medium" => "organic",
- "ga:source" => "google.com"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210131",
+ "ga:keyword" => "",
+ "ga:medium" => "organic",
+ "ga:source" => "google.com"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "paid",
- "ga:source" => "google.com"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "paid",
+ "ga:source" => "google.com"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "social",
- "ga:source" => "Twitter"
- },
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "newsletter",
- "ga:date" => "20210131",
- "ga:keyword" => "",
- "ga:medium" => "email",
- "ga:source" => "A Nice Newsletter"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "social",
+ "ga:source" => "Twitter"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "(none)",
- "ga:source" => "(direct)"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "newsletter",
+ "ga:date" => "20210131",
+ "ga:keyword" => "",
+ "ga:medium" => "email",
+ "ga:source" => "A Nice Newsletter"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "(none)",
+ "ga:source" => "(direct)"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_sources"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_sources"
)
- assert conn |> json_response(200) |> Enum.sort() == [
- %{"name" => "A Nice Newsletter", "visitors" => 1},
- %{"name" => "Direct / None", "visitors" => 1},
- %{"name" => "DuckDuckGo", "visitors" => 2},
- %{"name" => "Google", "visitors" => 4},
- %{"name" => "Twitter", "visitors" => 1}
- ]
- end
-
- test "UTM mediums data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- utm_medium: "social",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- utm_medium: "social",
- timestamp: ~N[2021-01-01 12:00:00]
- )
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "social",
- "ga:source" => "Twitter"
- },
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "(none)",
- "ga:source" => "(direct)"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true"
+ )
+
+ assert conn |> json_response(200) |> Enum.sort() == [
+ %{"name" => "A Nice Newsletter", "visitors" => 1},
+ %{"name" => "Direct / None", "visitors" => 1},
+ %{"name" => "DuckDuckGo", "visitors" => 2},
+ %{"name" => "Google", "visitors" => 4},
+ %{"name" => "Twitter", "visitors" => 1}
+ ]
+ end
+
+ test "UTM mediums data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ utm_medium: "social",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ utm_medium: "social",
+ timestamp: ~N[2021-01-01 12:00:00]
+ )
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "social",
+ "ga:source" => "Twitter"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "(none)",
+ "ga:source" => "(direct)"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_sources"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_sources"
)
- assert json_response(conn, 200) == [
- %{
- "bounce_rate" => 100.0,
- "name" => "social",
- "visit_duration" => 20,
- "visitors" => 3
- }
- ]
- end
-
- test "UTM campaigns data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "profile",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "social",
- "ga:source" => "Twitter"
- },
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "august",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "email",
- "ga:source" => "Gmail"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "bounce_rate" => 100.0,
+ "name" => "social",
+ "visit_duration" => 20,
+ "visitors" => 3
+ }
+ ]
+ end
+
+ test "UTM campaigns data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "profile",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "social",
+ "ga:source" => "Twitter"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "(not set)",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "email",
- "ga:source" => "Gmail"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "august",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "email",
+ "ga:source" => "Gmail"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "(not set)",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "email",
+ "ga:source" => "Gmail"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_sources"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_sources"
)
- assert json_response(conn, 200) == [
- %{
- "name" => "august",
- "visitors" => 2,
- "bounce_rate" => 50.0,
- "visit_duration" => 50.0
- },
- %{
- "name" => "profile",
- "visitors" => 2,
- "bounce_rate" => 100.0,
- "visit_duration" => 50.0
- }
- ]
- end
-
- test "UTM terms data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "oat milk",
- "ga:medium" => "paid",
- "ga:source" => "Google"
- },
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "Sweden",
- "ga:medium" => "paid",
- "ga:source" => "Google"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "name" => "august",
+ "visitors" => 2,
+ "bounce_rate" => 50.0,
+ "visit_duration" => 50.0
+ },
+ %{
+ "name" => "profile",
+ "visitors" => 2,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 50.0
+ }
+ ]
+ end
+
+ test "UTM terms data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "oat milk",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "(not set)",
- "ga:medium" => "paid",
- "ga:source" => "Google"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "Sweden",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:adContent" => "",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "(not set)",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_sources"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_sources"
)
- assert json_response(conn, 200) == [
- %{
- "name" => "Sweden",
- "visitors" => 3,
- "bounce_rate" => 67.0,
- "visit_duration" => 33.3
- },
- %{
- "name" => "oat milk",
- "visitors" => 2,
- "bounce_rate" => 100.0,
- "visit_duration" => 50.0
- }
- ]
- end
-
- test "UTM contents data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:adContent" => "ad",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "paid",
- "ga:source" => "Google"
- },
- metrics: %{
- "ga:bounces" => "1",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "blog",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "paid",
- "ga:source" => "Google"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "name" => "Sweden",
+ "visitors" => 3,
+ "bounce_rate" => 67.0,
+ "visit_duration" => 33.3
+ },
+ %{
+ "name" => "oat milk",
+ "visitors" => 2,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 50.0
+ }
+ ]
+ end
+
+ test "UTM contents data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:adContent" => "ad",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:adContent" => "(not set)",
- "ga:campaign" => "",
- "ga:date" => "20210101",
- "ga:keyword" => "",
- "ga:medium" => "paid",
- "ga:source" => "Google"
+ %{
+ dimensions: %{
+ "ga:adContent" => "blog",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "100",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:adContent" => "(not set)",
+ "ga:campaign" => "",
+ "ga:date" => "20210101",
+ "ga:keyword" => "",
+ "ga:medium" => "paid",
+ "ga:source" => "Google"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "100",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_sources"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_sources"
)
- assert json_response(conn, 200) == [
- %{
- "name" => "blog",
- "visitors" => 2,
- "bounce_rate" => 50.0,
- "visit_duration" => 50.0
- },
- %{
- "name" => "ad",
- "visitors" => 2,
- "bounce_rate" => 100.0,
- "visit_duration" => 50.0
- }
- ]
- end
-
- test "Page event data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- pathname: "/",
- hostname: "host-a.com",
- user_id: @user_id,
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- pathname: "/some-other-page",
- hostname: "host-a.com",
- user_id: @user_id,
- timestamp: ~N[2021-01-01 00:15:00]
- )
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:date" => "20210101",
- "ga:hostname" => "host-a.com",
- "ga:pagePath" => "/"
- },
- metrics: %{
- "ga:exits" => "0",
- "ga:pageviews" => "1",
- "ga:timeOnPage" => "700",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:date" => "20210101",
- "ga:hostname" => "host-b.com",
- "ga:pagePath" => "/some-other-page"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "name" => "blog",
+ "visitors" => 2,
+ "bounce_rate" => 50.0,
+ "visit_duration" => 50.0
+ },
+ %{
+ "name" => "ad",
+ "visitors" => 2,
+ "bounce_rate" => 100.0,
+ "visit_duration" => 50.0
+ }
+ ]
+ end
+
+ test "Page event data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ pathname: "/",
+ hostname: "host-a.com",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ pathname: "/some-other-page",
+ hostname: "host-a.com",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:15:00]
+ )
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:date" => "20210101",
+ "ga:hostname" => "host-a.com",
+ "ga:pagePath" => "/"
+ },
+ metrics: %{
+ "ga:exits" => "0",
+ "ga:pageviews" => "1",
+ "ga:timeOnPage" => "700",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:exits" => "1",
- "ga:pageviews" => "2",
- "ga:timeOnPage" => "60",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:date" => "20210101",
- "ga:hostname" => "host-b.com",
- "ga:pagePath" => "/some-other-page?wat=wot"
+ %{
+ dimensions: %{
+ "ga:date" => "20210101",
+ "ga:hostname" => "host-b.com",
+ "ga:pagePath" => "/some-other-page"
+ },
+ metrics: %{
+ "ga:exits" => "1",
+ "ga:pageviews" => "2",
+ "ga:timeOnPage" => "60",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:exits" => "0",
- "ga:pageviews" => "1",
- "ga:timeOnPage" => "60",
- "ga:users" => "1"
- }
- }
- ],
- site.id,
- "imported_pages"
- )
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101", "ga:landingPagePath" => "/"},
- metrics: %{
- "ga:bounces" => "1",
- "ga:entrances" => "3",
- "ga:sessionDuration" => "10",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:date" => "20210101",
+ "ga:hostname" => "host-b.com",
+ "ga:pagePath" => "/some-other-page?wat=wot"
+ },
+ metrics: %{
+ "ga:exits" => "0",
+ "ga:pageviews" => "1",
+ "ga:timeOnPage" => "60",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_entry_pages"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_pages"
)
- assert json_response(conn, 200) == [
- %{
- "bounce_rate" => nil,
- "time_on_page" => 60,
- "visitors" => 3,
- "pageviews" => 4,
- "name" => "/some-other-page"
- },
- %{
- "bounce_rate" => 25.0,
- "time_on_page" => 800.0,
- "visitors" => 2,
- "pageviews" => 2,
- "name" => "/"
- }
- ]
- end
-
- test "Exit page event data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- pathname: "/page1",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- pathname: "/page1",
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- pathname: "/page1",
- user_id: @user_id,
- timestamp: ~N[2021-01-01 00:00:00]
- ),
- build(:pageview,
- pathname: "/page2",
- user_id: @user_id,
- timestamp: ~N[2021-01-01 00:15:00]
- )
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:date" => "20210101",
- "ga:hostname" => "host-a.com",
- "ga:pagePath" => "/page2"
- },
- metrics: %{
- "ga:exits" => "0",
- "ga:pageviews" => "4",
- "ga:timeOnPage" => "10",
- "ga:users" => "2"
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:landingPagePath" => "/"},
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:entrances" => "3",
+ "ga:sessionDuration" => "10",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_pages"
- )
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101", "ga:exitPagePath" => "/page2"},
- metrics: %{"ga:exits" => "3", "ga:users" => "2"}
- }
- ],
- site.id,
- "imported_exit_pages"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_entry_pages"
)
- assert json_response(conn, 200) == [
- %{
- "name" => "/page2",
- "visitors" => 3,
- "visits" => 4,
- "exit_rate" => 80.0
- },
- %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66}
- ]
- end
-
- test "imports city data from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- country_code: "EE",
- timestamp: ~N[2021-01-01 00:15:00]
- ),
- build(:pageview,
- country_code: "EE",
- timestamp: ~N[2021-01-01 00:15:00]
- ),
- build(:pageview,
- country_code: "GB",
- timestamp: ~N[2021-01-01 00:15:00]
- )
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:countryIsoCode" => "EE",
- "ga:city" => "Tartu",
- "ga:date" => "20210101",
- "ga:regionIsoCode" => "Tartumaa"
- },
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{
- "ga:countryIsoCode" => "GB",
- "ga:city" => "Edinburgh",
- "ga:date" => "20210101",
- "ga:regionIsoCode" => "Midlothian"
- },
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "bounce_rate" => nil,
+ "time_on_page" => 60,
+ "visitors" => 3,
+ "pageviews" => 4,
+ "name" => "/some-other-page"
+ },
+ %{
+ "bounce_rate" => 25.0,
+ "time_on_page" => 800.0,
+ "visitors" => 2,
+ "pageviews" => 2,
+ "name" => "/"
+ }
+ ]
+ end
+
+ test "Exit page event data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ pathname: "/page1",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ pathname: "/page1",
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ pathname: "/page1",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:00:00]
+ ),
+ build(:pageview,
+ pathname: "/page2",
+ user_id: @user_id,
+ timestamp: ~N[2021-01-01 00:15:00]
+ )
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:date" => "20210101",
+ "ga:hostname" => "host-a.com",
+ "ga:pagePath" => "/page2"
+ },
+ metrics: %{
+ "ga:exits" => "0",
+ "ga:pageviews" => "4",
+ "ga:timeOnPage" => "10",
+ "ga:users" => "2"
+ }
}
- }
- ],
- site.id,
- "imported_locations"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_pages"
)
- assert json_response(conn, 200) == [
- %{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "🇪🇪"},
- %{
- "code" => 2_650_225,
- "name" => "Edinburgh",
- "visitors" => 1,
- "country_flag" => "🇬🇧"
- }
- ]
- end
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:exitPagePath" => "/page2"},
+ metrics: %{"ga:exits" => "3", "ga:users" => "2"}
+ }
+ ],
+ site.id,
+ import_id,
+ "imported_exit_pages"
+ )
- test "imports country data from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview,
- country_code: "EE",
- timestamp: ~N[2021-01-01 00:15:00]
- ),
- build(:pageview,
- country_code: "EE",
- timestamp: ~N[2021-01-01 00:15:00]
- ),
- build(:pageview,
- country_code: "GB",
- timestamp: ~N[2021-01-01 00:15:00]
- ),
- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:countryIsoCode" => "EE",
- "ga:city" => "Tartu",
- "ga:date" => "20210101",
- "ga:regionIsoCode" => "Tartumaa"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "name" => "/page2",
+ "visitors" => 3,
+ "visits" => 4,
+ "exit_rate" => 80.0
+ },
+ %{"name" => "/page1", "visitors" => 2, "visits" => 2, "exit_rate" => 66}
+ ]
+ end
+
+ test "imports city data from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview,
+ country_code: "EE",
+ timestamp: ~N[2021-01-01 00:15:00]
+ ),
+ build(:pageview,
+ country_code: "EE",
+ timestamp: ~N[2021-01-01 00:15:00]
+ ),
+ build(:pageview,
+ country_code: "GB",
+ timestamp: ~N[2021-01-01 00:15:00]
+ )
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:countryIsoCode" => "EE",
+ "ga:city" => "Tartu",
+ "ga:date" => "20210101",
+ "ga:regionIsoCode" => "Tartumaa"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:countryIsoCode" => "GB",
+ "ga:city" => "Edinburgh",
+ "ga:date" => "20210101",
+ "ga:regionIsoCode" => "Midlothian"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- },
- %{
- dimensions: %{
- "ga:countryIsoCode" => "GB",
- "ga:city" => "Edinburgh",
- "ga:date" => "20210101",
- "ga:regionIsoCode" => "Midlothian"
+ ],
+ site.id,
+ import_id,
+ "imported_locations"
+ )
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/cities?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{"code" => 588_335, "name" => "Tartu", "visitors" => 1, "country_flag" => "🇪🇪"},
+ %{
+ "code" => 2_650_225,
+ "name" => "Edinburgh",
+ "visitors" => 1,
+ "country_flag" => "🇬🇧"
+ }
+ ]
+ end
+
+ test "imports country data from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, import_id, [
+ build(:pageview,
+ country_code: "EE",
+ timestamp: ~N[2021-01-01 00:15:00]
+ ),
+ build(:pageview,
+ country_code: "EE",
+ timestamp: ~N[2021-01-01 00:15:00]
+ ),
+ build(:pageview,
+ country_code: "GB",
+ timestamp: ~N[2021-01-01 00:15:00]
+ ),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:countryIsoCode" => "EE",
+ "ga:city" => "Tartu",
+ "ga:date" => "20210101",
+ "ga:regionIsoCode" => "Tartumaa"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{
+ "ga:countryIsoCode" => "GB",
+ "ga:city" => "Edinburgh",
+ "ga:date" => "20210101",
+ "ga:regionIsoCode" => "Midlothian"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_locations"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_locations"
)
- assert json_response(conn, 200) == [
- %{
- "code" => "EE",
- "alpha_3" => "EST",
- "name" => "Estonia",
- "flag" => "🇪🇪",
- "visitors" => 3,
- "percentage" => 60
- },
- %{
- "code" => "GB",
- "alpha_3" => "GBR",
- "name" => "United Kingdom",
- "flag" => "🇬🇧",
- "visitors" => 2,
- "percentage" => 40
- }
- ]
- end
-
- test "Devices data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),
- build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),
- build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]),
- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
- ])
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "mobile"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "Laptop"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "code" => "EE",
+ "alpha_3" => "EST",
+ "name" => "Estonia",
+ "flag" => "🇪🇪",
+ "visitors" => 3,
+ "percentage" => 60
+ },
+ %{
+ "code" => "GB",
+ "alpha_3" => "GBR",
+ "name" => "United Kingdom",
+ "flag" => "🇬🇧",
+ "visitors" => 2,
+ "percentage" => 40
+ }
+ ]
+ end
+
+ test "Devices data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, import_id, [
+ build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "mobile"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:deviceCategory" => "Laptop"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_devices"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_devices"
)
- assert json_response(conn, 200) == [
- %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},
- %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},
- %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}
- ]
- end
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true"
+ )
- test "Browsers data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]),
- build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]),
- build(:imported_visitors, visitors: 2, date: ~D[2021-01-01])
- ])
-
- import_data(
- [
- %{
- dimensions: %{
- "ga:browser" => "User-Agent: Mozilla",
- "ga:date" => "20210101"
+ assert json_response(conn, 200) == [
+ %{"name" => "Desktop", "visitors" => 2, "percentage" => 40},
+ %{"name" => "Laptop", "visitors" => 2, "percentage" => 40},
+ %{"name" => "Mobile", "visitors" => 1, "percentage" => 20}
+ ]
+ end
+
+ test "Browsers data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, import_id, [
+ build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:imported_visitors, visitors: 2, date: ~D[2021-01-01])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{
+ "ga:browser" => "User-Agent: Mozilla",
+ "ga:date" => "20210101"
+ },
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ %{
+ dimensions: %{"ga:browser" => "Android Browser", "ga:date" => "20210101"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- },
- %{
- dimensions: %{"ga:browser" => "Android Browser", "ga:date" => "20210101"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- }
- ],
- site.id,
- "imported_browsers"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_browsers"
)
- assert stats = json_response(conn, 200)
- assert length(stats) == 3
- assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in stats
- assert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats
- assert %{"name" => "Chrome", "visitors" => 1, "percentage" => 25.0} in stats
- end
-
- test "OS data imported from Google Analytics", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),
- build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),
- build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]),
- build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
- ])
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Macintosh"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Linux"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:sessionDuration" => "10",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true"
+ )
+
+ assert stats = json_response(conn, 200)
+ assert length(stats) == 3
+ assert %{"name" => "Firefox", "visitors" => 2, "percentage" => 50.0} in stats
+ assert %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25.0} in stats
+ assert %{"name" => "Chrome", "visitors" => 1, "percentage" => 25.0} in stats
+ end
+
+ test "OS data imported from Google Analytics", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, import_id, [
+ build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]),
+ build(:imported_visitors, date: ~D[2021-01-01], visitors: 2)
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Macintosh"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210101", "ga:operatingSystem" => "Linux"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:sessionDuration" => "10",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_operating_systems"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_operating_systems"
)
- assert json_response(conn, 200) == [
- %{"name" => "Mac", "visitors" => 3, "percentage" => 60},
- %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}
- ]
- end
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true"
+ )
- test "Can import visit duration with scientific notation", %{conn: conn, site: site} do
- populate_stats(site, [
- build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
- build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
- ])
-
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101"},
- metrics: %{
- "ga:bounces" => "0",
- "ga:pageviews" => "1",
- "ga:sessionDuration" => "1.391607E7",
- "ga:sessions" => "1",
- "ga:users" => "1"
- }
- },
- %{
- dimensions: %{"ga:date" => "20210131"},
- metrics: %{
- "ga:bounces" => "1",
- "ga:pageviews" => "1",
- "ga:sessionDuration" => "60",
- "ga:sessions" => "1",
- "ga:users" => "1"
+ assert json_response(conn, 200) == [
+ %{"name" => "Mac", "visitors" => 3, "percentage" => 60},
+ %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40}
+ ]
+ end
+
+ test "Can import visit duration with scientific notation", %{
+ conn: conn,
+ site: site,
+ import_id: import_id
+ } do
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),
+ build(:pageview, timestamp: ~N[2021-01-31 00:00:00])
+ ])
+
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101"},
+ metrics: %{
+ "ga:bounces" => "0",
+ "ga:pageviews" => "1",
+ "ga:sessionDuration" => "1.391607E7",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "20210131"},
+ metrics: %{
+ "ga:bounces" => "1",
+ "ga:pageviews" => "1",
+ "ga:sessionDuration" => "60",
+ "ga:sessions" => "1",
+ "ga:users" => "1"
+ }
}
- }
- ],
- site.id,
- "imported_visitors"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_visitors"
)
- assert %{"top_stats" => top_stats} = json_response(conn, 200)
-
- visit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)
-
- assert visit_duration["value"] == 3_479_033
- end
-
- test "skips empty dates from import", %{conn: conn, site: site} do
- import_data(
- [
- %{
- dimensions: %{"ga:date" => "20210101"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
- }
- },
- %{
- dimensions: %{"ga:date" => "(other)"},
- metrics: %{
- "ga:users" => "1",
- "ga:pageviews" => "1",
- "ga:bounces" => "0",
- "ga:sessions" => "1",
- "ga:sessionDuration" => "60"
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"
+ )
+
+ assert %{"top_stats" => top_stats} = json_response(conn, 200)
+
+ visit_duration = Enum.find(top_stats, fn stat -> stat["name"] == "Visit duration" end)
+
+ assert visit_duration["value"] == 3_479_033
+ end
+
+ test "skips empty dates from import", %{conn: conn, site: site, import_id: import_id} do
+ import_data(
+ [
+ %{
+ dimensions: %{"ga:date" => "20210101"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
+ },
+ %{
+ dimensions: %{"ga:date" => "(other)"},
+ metrics: %{
+ "ga:users" => "1",
+ "ga:pageviews" => "1",
+ "ga:bounces" => "0",
+ "ga:sessions" => "1",
+ "ga:sessionDuration" => "60"
+ }
}
- }
- ],
- site.id,
- "imported_visitors"
- )
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"
+ ],
+ site.id,
+ import_id,
+ "imported_visitors"
)
- assert %{
- "top_stats" => [
- %{"name" => "Unique visitors", "value" => 1},
- %{"name" => "Total visits", "value" => 1},
- %{"name" => "Total pageviews", "value" => 1},
- %{"name" => "Views per visit", "value" => +0.0},
- %{"name" => "Bounce rate", "value" => 0},
- %{"name" => "Visit duration", "value" => 60}
- ]
- } = json_response(conn, 200)
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/top-stats?period=month&date=2021-01-01&with_imported=true"
+ )
+
+ assert %{
+ "top_stats" => [
+ %{"name" => "Unique visitors", "value" => 1},
+ %{"name" => "Total visits", "value" => 1},
+ %{"name" => "Total pageviews", "value" => 1},
+ %{"name" => "Views per visit", "value" => +0.0},
+ %{"name" => "Bounce rate", "value" => 0},
+ %{"name" => "Visit duration", "value" => 60}
+ ]
+ } = json_response(conn, 200)
+ end
end
end
end
diff --git a/test/plausible/imported/universal_analytics_test.exs b/test/plausible/imported/universal_analytics_test.exs
index 6d842be7a..80ad9caab 100644
--- a/test/plausible/imported/universal_analytics_test.exs
+++ b/test/plausible/imported/universal_analytics_test.exs
@@ -2,41 +2,58 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
use Plausible.DataCase, async: true
use Plausible.Test.Support.HTTPMocker
+ alias Plausible.Imported.SiteImport
alias Plausible.Imported.UniversalAnalytics
+ require Plausible.Imported.SiteImport
+
setup [:create_user, :create_new_site]
- describe "create_job/2 and parse_args/1" do
- test "parses job args properly" do
- site = insert(:site)
- site_id = site.id
+ describe "new_import/3 and parse_args/1" do
+ test "parses job args properly", %{user: user, site: site} do
expires_at = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
- job =
- UniversalAnalytics.create_job(site,
- view_id: 123,
- start_date: "2023-10-01",
- end_date: "2024-01-02",
- access_token: "access123",
- refresh_token: "refresh123",
- token_expires_at: expires_at
- )
+ assert {:ok, job} =
+ UniversalAnalytics.new_import(site, user,
+ view_id: 123,
+ start_date: "2023-10-01",
+ end_date: "2024-01-02",
+ access_token: "access123",
+ refresh_token: "refresh123",
+ token_expires_at: expires_at
+ )
+
+ assert %Oban.Job{
+ args:
+ %{
+ "import_id" => import_id,
+ "view_id" => 123,
+ "start_date" => "2023-10-01",
+ "end_date" => "2024-01-02",
+ "access_token" => "access123",
+ "refresh_token" => "refresh123",
+ "token_expires_at" => ^expires_at
+ } = args
+ } = Repo.reload!(job)
+
+ assert [
+ %{
+ id: ^import_id,
+ source: :universal_analytics,
+ start_date: ~D[2023-10-01],
+ end_date: ~D[2024-01-02],
+ status: SiteImport.pending()
+ }
+ ] = Plausible.Imported.list_all_imports(site)
- assert %Ecto.Changeset{
- data: %Oban.Job{},
- changes: %{
- args:
- %{
- "site_id" => ^site_id,
- "view_id" => 123,
- "start_date" => "2023-10-01",
- "end_date" => "2024-01-02",
- "access_token" => "access123",
- "refresh_token" => "refresh123",
- "token_expires_at" => ^expires_at
- } = args
+ assert %{
+ imported_data: %{
+ source: "Google Analytics",
+ start_date: ~D[2023-10-01],
+ end_date: ~D[2024-01-02],
+ status: "importing"
}
- } = job
+ } = Repo.reload!(site)
assert opts = [_ | _] = UniversalAnalytics.parse_args(args)
@@ -46,23 +63,34 @@ defmodule Plausible.Imported.UniversalAnalyticsTest do
end
end
- describe "import/2" do
+ describe "import_data/2" do
@tag :slow
- test "imports page views from Google Analytics", %{site: site} do
+ test "imports page views from Google Analytics", %{user: user, site: site} do
mock_http_with("google_analytics_import#1.json")
view_id = "54297898"
- date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
+ start_date = ~D[2011-01-01]
+ end_date = ~D[2022-07-19]
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()
- auth = {"***", "refresh_token", future}
+ access_token = "***"
+ refresh_token = "refresh_token"
- assert :ok ==
- UniversalAnalytics.import(site,
- date_range: date_range,
- view_id: view_id,
- auth: auth
- )
+ {:ok, job} =
+ UniversalAnalytics.new_import(
+ site,
+ user,
+ view_id: view_id,
+ start_date: start_date,
+ end_date: end_date,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ token_expires_at: future
+ )
+
+ job = Repo.reload!(job)
+
+ Plausible.Workers.ImportAnalytics.perform(job)
assert 1_495_150 == Plausible.Stats.Clickhouse.imported_pageview_count(site)
end
diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exs
index e62b29244..edb7d27fe 100644
--- a/test/plausible/purge_test.exs
+++ b/test/plausible/purge_test.exs
@@ -2,9 +2,28 @@ defmodule Plausible.PurgeTest do
use Plausible.DataCase
setup do
- site = insert(:site, stats_start_date: ~D[2020-01-01])
+ user = insert(:user)
+ site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])
- populate_stats(site, [
+ import_params = %{
+ source: :universal_analytics,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today()
+ }
+
+ [site_import1, site_import2] =
+ Enum.map(1..2, fn _ ->
+ site
+ |> Plausible.Imported.SiteImport.create_changeset(
+ user,
+ import_params
+ )
+ |> Plausible.Repo.insert!()
+ |> Plausible.Imported.SiteImport.complete_changeset()
+ |> Plausible.Repo.update!()
+ end)
+
+ populate_stats(site, site_import1.id, [
build(:pageview),
build(:imported_visitors),
build(:imported_sources),
@@ -17,13 +36,26 @@ defmodule Plausible.PurgeTest do
build(:imported_operating_systems)
])
- {:ok, %{site: site}}
+ populate_stats(site, site_import2.id, [
+ build(:pageview),
+ build(:imported_visitors),
+ build(:imported_sources),
+ build(:imported_pages),
+ build(:imported_entry_pages),
+ build(:imported_exit_pages),
+ build(:imported_locations),
+ build(:imported_devices),
+ build(:imported_browsers),
+ build(:imported_operating_systems)
+ ])
+
+ {:ok, %{site: site, site_import1: site_import1, site_import2: site_import2}}
end
test "delete_imported_stats!/1 deletes imported data", %{site: site} do
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
- assert await_clickhouse_count(query, 1)
+ assert await_clickhouse_count(query, 2)
end)
assert :ok == Plausible.Purge.delete_imported_stats!(site)
@@ -34,6 +66,29 @@ defmodule Plausible.PurgeTest do
end)
end
+ test "delete_imported_stats!/1 deletes imported data only for a particular site import", %{
+ site_import1: site_import1,
+ site_import2: site_import2
+ } do
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ query = from(imported in table, where: imported.import_id == ^site_import1.id)
+ assert await_clickhouse_count(query, 1)
+
+ query = from(imported in table, where: imported.import_id == ^site_import2.id)
+ assert await_clickhouse_count(query, 1)
+ end)
+
+ assert :ok == Plausible.Purge.delete_imported_stats!(site_import1)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ query = from(imported in table, where: imported.import_id == ^site_import1.id)
+ assert await_clickhouse_count(query, 0)
+
+ query = from(imported in table, where: imported.import_id == ^site_import2.id)
+ assert await_clickhouse_count(query, 1)
+ end)
+ end
+
test "delete_imported_stats!/1 resets stats_start_date", %{site: site} do
assert :ok == Plausible.Purge.delete_imported_stats!(site)
assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)
diff --git a/test/plausible/stats/interval_test.exs b/test/plausible/stats/interval_test.exs
index 893b7b8c6..36e3c6809 100644
--- a/test/plausible/stats/interval_test.exs
+++ b/test/plausible/stats/interval_test.exs
@@ -72,7 +72,7 @@ defmodule Plausible.Stats.IntervalTest do
describe "valid_for_period/3" do
test "common" do
- site = build(:site)
+ site = insert(:site)
assert valid_for_period?("month", "date", site: site)
refute valid_for_period?("30d", "month", site: site)
refute valid_for_period?("realtime", "week", site: site)
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index 118299443..697e34e20 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -120,7 +120,7 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows hourly if site is completely new", %{site: site} do
- site = Map.put(site, :stats_start_date, Timex.now())
+ site = Map.put(site, :stats_start_date, Timex.now() |> Timex.to_date())
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today()
@@ -130,7 +130,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows daily if site is more than a day old", %{site: site} do
- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1))
+ site =
+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(days: -1) |> Timex.to_date())
+
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today() |> Timex.shift(days: -1)
@@ -140,7 +142,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time shows monthly if site is more than a month old", %{site: site} do
- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))
+ site =
+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
+
q = Query.from(site, %{"period" => "all"})
assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)
@@ -150,7 +154,9 @@ defmodule Plausible.Stats.QueryTest do
end
test "all time uses passed interval different from the default interval", %{site: site} do
- site = Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1))
+ site =
+ Map.put(site, :stats_start_date, Timex.now() |> Timex.shift(months: -1) |> Timex.to_date())
+
q = Query.from(site, %{"period" => "all", "interval" => "week"})
assert q.date_range.first == Timex.today() |> Timex.shift(months: -1)
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index b08d878c9..538929ac8 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -8,6 +8,10 @@ defmodule PlausibleWeb.SiteControllerTest do
import Mox
import Plausible.Test.Support.HTML
+ alias Plausible.Imported.SiteImport
+
+ require Plausible.Imported.SiteImport
+
@v4_business_plan_id "857105"
setup :verify_on_exit!
@@ -1260,7 +1264,7 @@ defmodule PlausibleWeb.SiteControllerTest do
describe "POST /:website/settings/google-import" do
setup [:create_user, :log_in, :create_new_site]
- test "adds in-progress imported tag to site", %{conn: conn, site: site} do
+ test "creates site import instance", %{conn: conn, site: site} do
post(conn, "/#{site.domain}/settings/google-import", %{
"view_id" => "123",
"start_date" => "2018-03-01",
@@ -1270,12 +1274,11 @@ defmodule PlausibleWeb.SiteControllerTest do
"expires_at" => "2022-09-22T20:01:37.112777"
})
- imported_data = Repo.reload(site).imported_data
+ [site_import] = Plausible.Imported.list_all_imports(site)
- assert imported_data
- assert imported_data.source == "Google Analytics"
- assert imported_data.end_date == ~D[2022-03-01]
- assert imported_data.status == "importing"
+ assert site_import.source == :universal_analytics
+ assert site_import.end_date == ~D[2022-03-01]
+ assert site_import.status == SiteImport.pending()
end
test "schedules an import job in Oban", %{conn: conn, site: site} do
@@ -1288,11 +1291,12 @@ defmodule PlausibleWeb.SiteControllerTest do
"expires_at" => "2022-09-22T20:01:37.112777"
})
+ assert [%{id: import_id}] = Plausible.Imported.list_all_imports(site)
+
assert_enqueued(
worker: Plausible.Workers.ImportAnalytics,
args: %{
- "source" => "Google Analytics",
- "site_id" => site.id,
+ "import_id" => import_id,
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
@@ -1313,9 +1317,13 @@ defmodule PlausibleWeb.SiteControllerTest do
assert Repo.reload(site).imported_data == nil
end
- test "removes actual imported data from Clickhouse", %{conn: conn, site: site} do
- Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")
- |> Repo.update!()
+ test "removes actual imported data from Clickhouse", %{conn: conn, user: user, site: site} do
+ Plausible.Imported.NoopImporter.new_import(
+ site,
+ user,
+ start_date: ~D[2022-01-01],
+ end_date: Timex.today()
+ )
populate_stats(site, [
build(:imported_visitors, pageviews: 10)
@@ -1329,20 +1337,14 @@ defmodule PlausibleWeb.SiteControllerTest do
end)
end
- test "cancels Oban job if it exists", %{conn: conn, site: site} do
+ test "cancels Oban job if it exists", %{conn: conn, user: user, site: site} do
{:ok, job} =
- Plausible.Workers.ImportAnalytics.new(%{
- "source" => "Google Analytics",
- "site_id" => site.id,
- "view_id" => "123",
- "start_date" => "2022-01-01",
- "end_date" => "2023-01-01",
- "access_token" => "token"
- })
- |> Oban.insert()
-
- Plausible.Site.start_import(site, ~D[2022-01-01], Timex.today(), "Google Analytics")
- |> Repo.update!()
+ Plausible.Imported.NoopImporter.new_import(
+ site,
+ user,
+ start_date: ~D[2022-01-01],
+ end_date: Timex.today()
+ )
populate_stats(site, [
build(:imported_visitors, pageviews: 10)
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index 034447acf..ac21fc29c 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -145,6 +145,21 @@ defmodule Plausible.TestUtils do
:ok
end
+ def populate_stats(site, import_id, events) do
+ Enum.map(events, fn event ->
+ event = Map.put(event, :site_id, site.id)
+
+ case event do
+ %Plausible.ClickhouseEventV2{} ->
+ event
+
+ imported_event ->
+ Map.put(imported_event, :import_id, import_id)
+ end
+ end)
+ |> populate_stats
+ end
+
def populate_stats(site, events) do
Enum.map(events, fn event ->
Map.put(event, :site_id, site.id)
diff --git a/test/workers/import_analytics_test.exs b/test/workers/import_analytics_test.exs
index cb9782ccb..fe7f1ad39 100644
--- a/test/workers/import_analytics_test.exs
+++ b/test/workers/import_analytics_test.exs
@@ -2,64 +2,77 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
use Plausible.DataCase
use Bamboo.Test
+ alias Plausible.Imported.SiteImport
alias Plausible.Workers.ImportAnalytics
+ require Plausible.Imported.SiteImport
+
@moduletag capture_log: true
- describe "Universal Analytics" do
+ describe "perform/1" do
setup do
%{
- imported_data: %Plausible.Site.ImportedData{
+ import_opts: [
start_date: Timex.today() |> Timex.shift(days: -7),
- end_date: Timex.today(),
- source: "Noop",
- status: "importing"
- }
+ end_date: Timex.today()
+ ]
}
end
- test "updates the imported_data field for the site after successful import", %{
- imported_data: imported_data
+ test "updates site import after successful import", %{
+ import_opts: import_opts
} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user])
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
+
+ assert [%{status: SiteImport.pending()}] = Plausible.Imported.list_all_imports(site)
+
+ # before_start callback triggered
+ assert_received {:before_start, import_id}
+
+ assert :ok =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id
- }
- })
+ assert [%{id: ^import_id, status: SiteImport.completed()}] =
+ Plausible.Imported.list_all_imports(site)
- assert Repo.reload!(site).imported_data.status == "ok"
+ # on_success callback triggered
+ assert_received {:on_success, ^import_id}
end
- test "updates the stats_start_date field for the site after successful import", %{
- imported_data: imported_data
+ test "clears stats_start_date field for the site after successful import", %{
+ import_opts: import_opts
} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user], stats_start_date: ~D[2005-01-01])
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id
- }
- })
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
- assert Repo.reload!(site).stats_start_date == imported_data.start_date
+ assert :ok =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+
+ site = Repo.reload!(site)
+ assert site.stats_start_date == nil
+ assert Plausible.Sites.stats_start_date(site) == import_opts[:start_date]
+ assert Repo.reload!(site).stats_start_date == import_opts[:start_date]
end
- test "sends email to owner after successful import", %{imported_data: imported_data} do
+ test "sends email to owner after successful import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user])
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id
- }
- })
+ assert :ok =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
assert_email_delivered_with(
to: [user],
@@ -67,54 +80,57 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
)
end
- test "updates site record after failed import", %{imported_data: imported_data} do
+ test "updates site import record after failed import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user])
+ import_opts = Keyword.put(import_opts, :error, true)
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id,
- "error" => true
- }
- })
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
- assert Repo.reload!(site).imported_data.status == "error"
+ assert {:discard, "Something went wrong"} =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+
+ assert [%{status: SiteImport.failed()}] = Plausible.Imported.list_all_imports(site)
end
- test "clears any orphaned data during import", %{imported_data: imported_data} do
+ test "clears any orphaned data during import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user])
+ import_opts = Keyword.put(import_opts, :error, true)
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
populate_stats(site, [
- build(:imported_visitors, pageviews: 10)
+ build(:imported_visitors, import_id: job.args.import_id, pageviews: 10)
])
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id,
- "error" => true
- }
- })
+ assert {:discard, _} =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
assert eventually(fn ->
count = Plausible.Stats.Clickhouse.imported_pageview_count(site)
{count == 0, count}
end)
+
+ # on_failure callback triggered
+ assert_received {:on_failure, _import_id}
end
- test "sends email to owner after failed import", %{imported_data: imported_data} do
+ test "sends email to owner after failed import", %{import_opts: import_opts} do
user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
- site = insert(:site, members: [user], imported_data: imported_data)
+ site = insert(:site, members: [user])
+ import_opts = Keyword.put(import_opts, :error, true)
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
- ImportAnalytics.perform(%Oban.Job{
- args: %{
- "source" => "Noop",
- "site_id" => site.id,
- "error" => true
- }
- })
+ assert {:discard, _} =
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
assert_email_delivered_with(
to: [user],
@@ -122,4 +138,96 @@ defmodule Plausible.Workers.ImportAnalyticsTest do
)
end
end
+
+ describe "perform/1 notifications" do
+ setup do
+ on_exit(fn ->
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
+ Repo.delete_all(Plausible.Site)
+ Repo.delete_all(Plausible.Auth.User)
+ end)
+
+ :ok
+ end)
+
+ %{
+ import_opts: [
+ start_date: Timex.today() |> Timex.shift(days: -7),
+ end_date: Timex.today()
+ ]
+ }
+ end
+
+ test "sends oban notification to calling process on completion when instructed", %{
+ import_opts: import_opts
+ } do
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+ site = insert(:site, members: [user])
+ import_opts = Keyword.put(import_opts, :listen?, true)
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
+ import_id = job.args[:import_id]
+
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+
+ assert_receive {:notification, :analytics_imports_jobs, %{"complete" => ^import_id}}
+ end)
+ end
+
+ test "sends oban notification to calling process on permanent failure when instructed", %{
+ import_opts: import_opts
+ } do
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+ site = insert(:site, members: [user])
+
+ import_opts =
+ import_opts
+ |> Keyword.put(:listen?, true)
+ |> Keyword.put(:error, true)
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
+ import_id = job.args[:import_id]
+
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+
+ assert_receive {:notification, :analytics_imports_jobs, %{"fail" => ^import_id}}
+ end)
+ end
+
+ test "sends oban notification to calling process on transient failure when instructed", %{
+ import_opts: import_opts
+ } do
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Plausible.Repo, fn ->
+ user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1))
+ site = insert(:site, members: [user])
+
+ import_opts =
+ import_opts
+ |> Keyword.put(:listen?, true)
+ |> Keyword.put(:crash, true)
+
+ {:ok, job} = Plausible.Imported.NoopImporter.new_import(site, user, import_opts)
+ import_id = job.args[:import_id]
+
+ site_import = Repo.get(Plausible.Imported.SiteImport, import_id)
+
+ # emulate oban error handler
+ try do
+ job
+ |> Repo.reload!()
+ |> ImportAnalytics.perform()
+ rescue
+ _ -> ImportAnalytics.import_fail_transient(site_import)
+ end
+
+ assert_receive {:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}}
+ end)
+ end
+ end
end