Compare commits
18 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
23c4603456 | 3 years ago |
|
|
40b42ad4f8 | 3 years ago |
|
|
ebb19a6053 | 3 years ago |
|
|
d4878beabd | 3 years ago |
|
|
45afa35813 | 3 years ago |
|
|
9fedb0be30 | 3 years ago |
|
|
7593cd4b8e | 3 years ago |
|
|
325bbe7c41 | 3 years ago |
|
|
53c2ea9e90 | 3 years ago |
|
|
80db5cc5d7 | 3 years ago |
|
|
386b0d5d9f | 3 years ago |
|
|
fcfb54dde0 | 3 years ago |
|
|
5fcee362b0 | 3 years ago |
|
|
003a996c31 | 3 years ago |
|
|
f280c52bd3 | 3 years ago |
|
|
2c67d41b8f | 3 years ago |
|
|
93f9fe0481 | 3 years ago |
|
|
167a084673 | 3 years ago |
@ -1,38 +0,0 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240302"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,38 +0,0 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20200421"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20170118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,188 +0,0 @@
|
||||
defmodule Plausible.DataMigration.SiteImports do
|
||||
@moduledoc """
|
||||
Site imports migration backfilling SiteImport entries for old imports
|
||||
and alters import end dates to match actual end date of respective import stats.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.ClickhouseRepo
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
def run(opts \\ []) do
|
||||
dry_run? = Keyword.get(opts, :dry_run?, true)
|
||||
|
||||
site_import_query =
|
||||
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
|
||||
|
||||
sites_with_imports =
|
||||
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|
||||
|> Repo.all(log: false)
|
||||
|
||||
sites_count = length(sites_with_imports)
|
||||
|
||||
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
|
||||
|
||||
for {site, idx} <- Enum.with_index(sites_with_imports) do
|
||||
site_imports =
|
||||
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all(log: false)
|
||||
|
||||
IO.puts(
|
||||
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
|
||||
)
|
||||
|
||||
site_imports =
|
||||
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
|
||||
IO.puts("Creating legacy site import entry for site ID #{site.id}")
|
||||
|
||||
# create legacy entry if there's not one yet
|
||||
params =
|
||||
site.imported_data
|
||||
|> Imported.SiteImport.from_legacy()
|
||||
|> Map.put(:site_id, site.id)
|
||||
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
|
||||
|
||||
legacy_site_import =
|
||||
%Imported.SiteImport{}
|
||||
|> Ecto.Changeset.change(params)
|
||||
|> insert!(dry_run?)
|
||||
|
||||
[legacy_site_import | site_imports]
|
||||
else
|
||||
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
|
||||
|
||||
site_imports
|
||||
end
|
||||
|
||||
# adjust end date for each site import
|
||||
for site_import <- site_imports do
|
||||
IO.puts(
|
||||
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
|
||||
)
|
||||
|
||||
import_ids =
|
||||
if site_import.legacy do
|
||||
[0, site_import.id]
|
||||
else
|
||||
[site_import.id]
|
||||
end
|
||||
|
||||
end_date = imported_stats_end_date(site.id, import_ids)
|
||||
|
||||
if !end_date do
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
|
||||
)
|
||||
|
||||
if site_import.legacy do
|
||||
clear_imported_data(site, dry_run?)
|
||||
end
|
||||
|
||||
delete!(site_import, dry_run?)
|
||||
else
|
||||
end_date =
|
||||
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
|
||||
end_date
|
||||
else
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
|
||||
)
|
||||
|
||||
site_import.end_date
|
||||
end
|
||||
|
||||
site_import
|
||||
|> Ecto.Changeset.change(end_date: end_date)
|
||||
|> update!(dry_run?)
|
||||
|
||||
IO.puts(
|
||||
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Done processing site ID #{site.id}")
|
||||
end
|
||||
|
||||
IO.puts("Finished")
|
||||
end
|
||||
|
||||
# Exposed for testing purposes
|
||||
@doc false
|
||||
def imported_stats_end_date(site_id, import_ids) do
|
||||
[first_schema | schemas] = Imported.schemas()
|
||||
|
||||
query =
|
||||
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
|
||||
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
|
||||
end)
|
||||
|
||||
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
|
||||
|
||||
if dates != [] do
|
||||
case Enum.max(dates, Date) do
|
||||
# no stats for this domain yet
|
||||
~D[1970-01-01] ->
|
||||
nil
|
||||
|
||||
date ->
|
||||
date
|
||||
end
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp insert!(changeset, false = _dry_run?) do
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
defp insert!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
changeset
|
||||
|> Ecto.Changeset.change(id: 0)
|
||||
|> Ecto.Changeset.apply_changes()
|
||||
else
|
||||
raise "Invalid insert: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, false = _dry_run?) do
|
||||
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, true = _dry_run?) do
|
||||
%{site | imported_data: nil}
|
||||
end
|
||||
|
||||
defp update!(changeset, false = _dry_run?) do
|
||||
Repo.update!(changeset)
|
||||
end
|
||||
|
||||
defp update!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
Ecto.Changeset.apply_changes(changeset)
|
||||
else
|
||||
raise "Invalid update: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp delete!(entity, false = _dry_run?) do
|
||||
Repo.delete!(entity)
|
||||
end
|
||||
|
||||
defp delete!(entity, true = _dry_run?) do
|
||||
entity
|
||||
end
|
||||
|
||||
defp max_date_query(schema, site_id, import_ids) do
|
||||
from(q in schema,
|
||||
where: q.site_id == ^site_id,
|
||||
where: q.import_id in ^import_ids,
|
||||
select: %{max_date: fragment("max(?)", q.date)}
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -1,111 +0,0 @@
|
||||
defmodule Plausible.Stats.TableDecider do
|
||||
@moduledoc """
|
||||
This module contains logic for deciding which tables need to be queried given a query
|
||||
and metrics, with the purpose of reducing the number of queries and JOINs needed to perform.
|
||||
"""
|
||||
|
||||
import Enum, only: [empty?: 1]
|
||||
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def events_join_sessions?(query) do
|
||||
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
|
||||
end
|
||||
|
||||
def partition_metrics(metrics, query, breakdown_property \\ nil) do
|
||||
%{
|
||||
event: event_only_metrics,
|
||||
session: session_only_metrics,
|
||||
either: either_metrics,
|
||||
other: other_metrics,
|
||||
sample_percent: sample_percent
|
||||
} =
|
||||
partition(metrics, query, &metric_partitioner/2)
|
||||
|
||||
# Treat breakdown property as yet another filter
|
||||
filters =
|
||||
if breakdown_property do
|
||||
Map.put(query.filters, breakdown_property, nil)
|
||||
else
|
||||
query.filters
|
||||
end
|
||||
|
||||
%{event: event_only_filters, session: session_only_filters} =
|
||||
partition(filters, query, &filters_partitioner/2)
|
||||
|
||||
cond do
|
||||
# Only one table needs to be queried
|
||||
empty?(event_only_metrics) && empty?(event_only_filters) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) && empty?(session_only_filters) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Filters on both events and sessions, but only one kind of metric
|
||||
empty?(event_only_metrics) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Default: prefer sessions
|
||||
true ->
|
||||
{event_only_metrics ++ sample_percent,
|
||||
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
end
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, :conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :average_revenue), do: :event
|
||||
defp metric_partitioner(_, :total_revenue), do: :event
|
||||
defp metric_partitioner(_, :pageviews), do: :event
|
||||
defp metric_partitioner(_, :events), do: :event
|
||||
defp metric_partitioner(_, :bounce_rate), do: :session
|
||||
defp metric_partitioner(_, :visit_duration), do: :session
|
||||
defp metric_partitioner(_, :views_per_visit), do: :session
|
||||
|
||||
# Metrics which used to only be queried from one table but can be calculated from either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visits), do: :either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visitors), do: :either
|
||||
|
||||
defp metric_partitioner(_, :visits), do: :session
|
||||
defp metric_partitioner(_, :visitors), do: :event
|
||||
# Calculated metrics - handled on callsite separately from other metrics.
|
||||
defp metric_partitioner(_, :time_on_page), do: :other
|
||||
defp metric_partitioner(_, :total_visitors), do: :other
|
||||
defp metric_partitioner(_, :percentage), do: :other
|
||||
# Sample percentage is included in both tables if queried.
|
||||
defp metric_partitioner(_, :sample_percent), do: :sample_percent
|
||||
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
|
||||
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, _), do: :either
|
||||
|
||||
defp filters_partitioner(_, {"event:" <> _, _}), do: :event
|
||||
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
|
||||
do: :either
|
||||
|
||||
defp filters_partitioner(_, {"visit:" <> _, _}),
|
||||
do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
|
||||
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp filters_partitioner(_, _), do: :either
|
||||
|
||||
@default %{event: [], session: [], either: [], other: [], sample_percent: []}
|
||||
defp partition(values, query, partitioner) do
|
||||
Enum.reduce(values, @default, fn value, acc ->
|
||||
key = partitioner.(query, value)
|
||||
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -1,35 +0,0 @@
|
||||
defmodule Plausible.IngestRepo.Migrations.AddMetricsToImportedTables do
|
||||
use Ecto.Migration
|
||||
|
||||
@add_pageviews_for [
|
||||
"imported_browsers",
|
||||
"imported_devices",
|
||||
"imported_entry_pages",
|
||||
"imported_exit_pages",
|
||||
"imported_locations",
|
||||
"imported_operating_systems",
|
||||
"imported_sources"
|
||||
]
|
||||
|
||||
def up do
|
||||
for table <- @add_pageviews_for do
|
||||
add_column(table, "pageviews", "UInt64")
|
||||
end
|
||||
|
||||
add_column("imported_browsers", "browser_version", "String")
|
||||
add_column("imported_exit_pages", "bounces", "UInt32")
|
||||
add_column("imported_exit_pages", "visit_duration", "UInt64")
|
||||
add_column("imported_operating_systems", "operating_system_version", "String")
|
||||
add_column("imported_pages", "visits", "UInt64")
|
||||
add_column("imported_sources", "referrer", "String")
|
||||
add_column("imported_sources", "utm_source", "String")
|
||||
end
|
||||
|
||||
defp add_column(table, column, type) do
|
||||
execute """
|
||||
ALTER TABLE #{table}
|
||||
#{Plausible.MigrationUtils.on_cluster_statement(table)}
|
||||
ADD COLUMN #{column} #{type}
|
||||
"""
|
||||
end
|
||||
end
|
||||
@ -1,19 +0,0 @@
|
||||
defmodule Plausible.ClickhouseRepo.Migrations.HostnamesInSessions do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
execute """
|
||||
ALTER TABLE sessions_v2
|
||||
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
|
||||
ADD COLUMN exit_page_hostname String CODEC(ZSTD(3))
|
||||
"""
|
||||
end
|
||||
|
||||
def down do
|
||||
execute """
|
||||
ALTER TABLE sessions_v2
|
||||
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
|
||||
DROP COLUMN exit_page_hostname
|
||||
"""
|
||||
end
|
||||
end
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1 +1 @@
|
||||
6.3.0
|
||||
6.2.1
|
||||
|
||||
@ -1,204 +0,0 @@
|
||||
defmodule Plausible.DataMigration.SiteImportsTest do
|
||||
use Plausible.DataCase, async: true
|
||||
|
||||
import ExUnit.CaptureIO
|
||||
|
||||
alias Plausible.DataMigration.SiteImports
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
describe "run/1" do
|
||||
test "runs for empty dataset" do
|
||||
dry_run_output =
|
||||
capture_io(fn ->
|
||||
assert :ok = SiteImports.run()
|
||||
end)
|
||||
|
||||
assert dry_run_output =~ "Processing 0 sites"
|
||||
assert dry_run_output =~ "DRY RUN: true"
|
||||
|
||||
real_run_output =
|
||||
capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end)
|
||||
|
||||
assert real_run_output =~ "Processing 0 sites"
|
||||
assert real_run_output =~ "DRY RUN: false"
|
||||
end
|
||||
|
||||
test "adds site import entry when it's missing and adjusts end date" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-07])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id > 0
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-07]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
|
||||
test "runs in dry mode without making any persistent changes" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-07])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run()
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)
|
||||
assert id == 0
|
||||
end
|
||||
|
||||
test "does not set end date to latter than the current one" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-10])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id > 0
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-08]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
|
||||
test "removes site import when there are no stats" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
_another_site_import = insert(:site_import, site: site)
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 site"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
assert [] = Imported.list_all_imports(site)
|
||||
end
|
||||
|
||||
test "leaves site and imports unchanged if everything fits" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
existing_import =
|
||||
insert(:site_import,
|
||||
site: site,
|
||||
start_date: ~D[2021-01-02],
|
||||
end_date: ~D[2021-01-08],
|
||||
status: :completed,
|
||||
legacy: true
|
||||
)
|
||||
|
||||
populate_stats(site, existing_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-08])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id == existing_import.id
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-08]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
end
|
||||
|
||||
describe "imported_stats_end_date/1" do
|
||||
test "returns nil when there are no stats" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) == nil
|
||||
end
|
||||
|
||||
test "returns date when there are stats recorded in one imported table" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-01])
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
~D[2021-01-01]
|
||||
end
|
||||
|
||||
test "returns max date across all imported tables" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-01]),
|
||||
build(:imported_visitors, date: ~D[2021-01-07]),
|
||||
build(:imported_sources, date: ~D[2021-01-01]),
|
||||
build(:imported_sources, date: ~D[2021-01-08]),
|
||||
build(:imported_entry_pages, date: ~D[2021-01-02]),
|
||||
build(:imported_entry_pages, date: ~D[2021-02-11]),
|
||||
build(:imported_exit_pages, date: ~D[2021-01-01]),
|
||||
build(:imported_exit_pages, date: ~D[2021-01-08]),
|
||||
build(:imported_locations, date: ~D[2021-01-01]),
|
||||
build(:imported_locations, date: ~D[2021-01-08]),
|
||||
build(:imported_devices, date: ~D[2021-01-01]),
|
||||
build(:imported_devices, date: ~D[2021-01-08]),
|
||||
build(:imported_browsers, date: ~D[2021-01-01]),
|
||||
build(:imported_browsers, date: ~D[2021-01-08]),
|
||||
build(:imported_operating_systems, date: ~D[2021-01-01]),
|
||||
build(:imported_operating_systems, date: ~D[2021-01-08])
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
~D[2021-02-11]
|
||||
end
|
||||
|
||||
test "considers all imported tables" do
|
||||
date = ~D[2021-01-11]
|
||||
|
||||
for {table, idx} <- Enum.with_index(Imported.tables()) do
|
||||
site_import = insert(:site_import)
|
||||
end_date = Date.add(date, idx)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(String.to_atom(table), date: date),
|
||||
build(String.to_atom(table), date: end_date)
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
end_date
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,164 +0,0 @@
|
||||
defmodule Plausible.Stats.TableDeciderTest do
|
||||
use Plausible.DataCase, async: true
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
import Plausible.Stats.TableDecider
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins disabled" do
|
||||
assert not events_join_sessions?(make_query(false, %{}))
|
||||
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
|
||||
assert events_join_sessions?(make_query(false, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins enabled" do
|
||||
assert not events_join_sessions?(make_query(true, %{}))
|
||||
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
|
||||
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
describe "partition_metrics" do
|
||||
test "with no metrics or filters" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([], query) == {[], [], []}
|
||||
end
|
||||
|
||||
test "session-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
|
||||
{[], [:bounce_rate, :views_per_visit], []}
|
||||
end
|
||||
|
||||
test "event-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, event-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, session-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
|
||||
end
|
||||
|
||||
test "session filters but no session metrics" do
|
||||
query = make_query(false, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "sample_percent is added to both types of metrics" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [], []}
|
||||
|
||||
assert partition_metrics([:bounce_rate, :sample_percent], query) ==
|
||||
{[], [:bounce_rate, :sample_percent], []}
|
||||
|
||||
assert partition_metrics([:total_revenue, :bounce_rate, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [:bounce_rate, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "other metrics put in its own result" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
|
||||
{[], [], [:time_on_page, :percentage, :total_visitors]}
|
||||
end
|
||||
|
||||
test "raises if unknown metric" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert_raise ArgumentError, fn ->
|
||||
partition_metrics([:foobar], query)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "partition_metrics with experimental_reduced_joins enabled" do
|
||||
test "metrics that can be calculated on either when event-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
|
||||
assert partition_metrics([:pageviews, :visits], query) == {[:pageviews, :visits], [], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either when session-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :visitors], query) ==
|
||||
{[], [:bounce_rate, :visitors], []}
|
||||
|
||||
assert partition_metrics([:visit_duration, :visits], query) ==
|
||||
{[], [:visit_duration, :visits], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either are biased to sessions" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
|
||||
{[:total_revenue], [:bounce_rate, :visitors], []}
|
||||
end
|
||||
|
||||
test "sample_percent is handled with either metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors, :sample_percent], query) ==
|
||||
{[], [:visitors, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events" do
|
||||
query = make_query(true, %{name: "pageview"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events and sessions" do
|
||||
query = make_query(true, %{name: "pageview", exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on either" do
|
||||
query = make_query(true, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on sessions" do
|
||||
query = make_query(true, %{exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "breakdown value leans metric" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
|
||||
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
|
||||
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
|
||||
end
|
||||
end
|
||||
|
||||
defp make_query(experimental_reduced_joins?, filters) do
|
||||
Query.from(build(:site), %{
|
||||
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
|
||||
"filters" => Jason.encode!(filters)
|
||||
})
|
||||
end
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue