Support `with_imported=true` in Stats API aggregate (#2919)

* default to source_query.include_imported before true

This fixes a bug where a Stats API aggregate query was trying
to query current period *without imported data* but previous
period *with imported data*.

* remove __internal_visits at a better time

As soon as we have made the db query, we don't need it anymore

* disallow events metric with imported data

* update changelog

* keep default in a better place

* remove unused option

* update test description

Co-authored-by: Vini Brasil <vini@hey.com>

---------

Co-authored-by: Vini Brasil <vini@hey.com>
monetary-values r2023.05.18-114
RobertJoonas 4 years ago committed by GitHub
parent f29d07a49e
commit fa54efbc6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
## Unreleased
### Added
- Add support for `with_imported=true` in Stats API aggregate endpoint
- Ability to use '--' instead of '=' sign in the `tagged-events` classnames
- 'Last updated X seconds ago' info to 'current visitors' tooltips
- Add support for more Bamboo adapters, i.e. `Bamboo.MailgunAdapter`, `Bamboo.MandrillAdapter`, `Bamboo.SendGridAdapter` plausible/analytics#2649

@ -1,7 +1,7 @@
defmodule Plausible.Stats.Aggregate do
alias Plausible.Stats.Query
use Plausible.ClickhouseRepo
import Plausible.Stats.{Base, Imported}
import Plausible.Stats.{Base, Imported, Util}
@event_metrics [:visitors, :pageviews, :events, :sample_percent]
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
@ -43,6 +43,7 @@ defmodule Plausible.Stats.Aggregate do
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, :aggregate, metrics)
|> ClickhouseRepo.one()
|> remove_internal_visits_metric()
end
defp aggregate_time_on_page(site, query) do

@ -53,20 +53,16 @@ defmodule Plausible.Stats.Comparisons do
it will be shifted to start on Sunday, January 2nd, 2022 instead of
January 1st. Defaults to false.
* `:include_imported?` - includes imported data in the generated comparison
query if there is imported data. Defaults to true.
"""
def compare(%Plausible.Site{} = site, %Stats.Query{} = source_query, mode, opts \\ []) do
opts =
opts
|> Keyword.put_new(:now, Timex.now(site.timezone))
|> Keyword.put_new(:match_day_of_week?, false)
|> Keyword.put_new(:include_imported?, true)
with :ok <- validate_mode(source_query, mode),
{:ok, comparison_query} <- do_compare(source_query, mode, opts),
comparison_query <- maybe_include_imported(comparison_query, site, opts),
comparison_query <- maybe_include_imported(comparison_query, source_query, site),
do: {:ok, comparison_query}
end
@ -165,8 +161,9 @@ defmodule Plausible.Stats.Comparisons do
Date.add(date, -days_to_subtract)
end
defp maybe_include_imported(query, site, opts) do
include_imported? = Stats.Query.include_imported?(query, site, opts[:include_imported?])
defp maybe_include_imported(query, source_query, site) do
requested? = source_query.imported_data_requested
include_imported? = Stats.Query.include_imported?(query, site, requested?)
%Stats.Query{query | include_imported: include_imported?}
end

@ -4,6 +4,7 @@ defmodule Plausible.Stats.Query do
period: nil,
filters: %{},
sample_threshold: 20_000_000,
imported_data_requested: false,
include_imported: false
@default_sample_threshold 20_000_000
@ -20,8 +21,7 @@ defmodule Plausible.Stats.Query do
interval: params["interval"] || Interval.default_for_period(params["period"]),
date_range: Date.range(date, date),
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold),
include_imported: false
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
end
@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "7d"} = params) do
@ -49,7 +49,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "30d"} = params) do
@ -63,7 +63,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "month"} = params) do
@ -79,7 +79,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "6mo"} = params) do
@ -98,7 +98,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "12mo"} = params) do
@ -117,7 +117,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "year"} = params) do
@ -134,7 +134,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(site, %{"period" => "all"} = params) do
@ -194,7 +194,7 @@ defmodule Plausible.Stats.Query do
filters: FilterParser.parse_filters(params["filters"]),
sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold)
}
|> maybe_include_imported(site, params)
|> put_imported_opts(site, params)
end
def from(tz, params) do
@ -248,9 +248,12 @@ defmodule Plausible.Stats.Query do
end
end
defp maybe_include_imported(query, site, params) do
defp put_imported_opts(query, site, params) do
requested? = params["with_imported"] == "true"
%{query | include_imported: include_imported?(query, site, requested?)}
query
|> Map.put(:imported_data_requested, requested?)
|> Map.put(:include_imported, include_imported?(query, site, requested?))
end
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()

@ -8,12 +8,16 @@ defmodule Plausible.Stats.Util do
is needed to calculate these from imported data. This function removes that metric
from all entries in the results list.
"""
def remove_internal_visits_metric(results, metrics) do
def remove_internal_visits_metric(results, metrics) when is_list(results) do
if :bounce_rate in metrics or :visit_duration in metrics do
results
|> Enum.map(&Map.delete(&1, :__internal_visits))
|> Enum.map(&remove_internal_visits_metric/1)
else
results
end
end
def remove_internal_visits_metric(result) when is_map(result) do
Map.delete(result, :__internal_visits)
end
end

@ -42,10 +42,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
Plausible.Stats.aggregate(site, query, metrics)
end
results =
results
|> Map.take(metrics)
json(conn, %{results: results})
else
{:error, msg} ->
@ -141,6 +137,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end)
end
defp validate_metric("events", nil, %{include_imported: true}) do
{:error, "Metric `events` cannot be queried with imported data"}
end
defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric}
defp validate_metric(metric, property, query) when metric in @session_metrics do

@ -1304,8 +1304,7 @@ defmodule PlausibleWeb.Api.StatsController do
[
from: params["compare_from"],
to: params["compare_to"],
match_day_of_week?: params["match_day_of_week"] == "true",
include_imported?: params["with_imported"] == "true"
match_day_of_week?: params["match_day_of_week"] == "true"
]
end

@ -1,6 +1,7 @@
defmodule Plausible.Stats.ComparisonsTest do
use Plausible.DataCase
alias Plausible.Stats.{Query, Comparisons}
import Plausible.TestUtils
describe "with period set to this month" do
test "shifts back this month period when mode is previous_period" do
@ -200,4 +201,16 @@ defmodule Plausible.Stats.ComparisonsTest do
Comparisons.compare(site, query, "custom", from: "2022-05-30", to: "2022-05-25")
end
end
describe "include_imported" do
setup [:create_user, :create_new_site, :add_imported_data]
test "defaults to source_query.include_imported", %{site: site} do
query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"})
assert query.include_imported == false
{:ok, comparison_query} = Comparisons.compare(site, query, "previous_period")
assert comparison_query.include_imported == false
end
end
end

@ -1,5 +1,6 @@
defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
use PlausibleWeb.ConnCase
import Plausible.TestUtils
setup [:create_user, :create_new_site, :create_api_key, :use_api_key]
@user_id 123
@ -250,6 +251,116 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
end
end
describe "with imported data" do
setup :add_imported_data
test "does not count imported stats unless specified", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:pageview, timestamp: ~N[2023-01-01 00:00:00])
])
query_params = %{
"site_id" => site.domain,
"period" => "day",
"date" => "2023-01-01",
"metrics" => "pageviews"
}
conn1 = get(conn, "/api/v1/stats/aggregate", query_params)
assert json_response(conn1, 200)["results"] == %{
"pageviews" => %{"value" => 1}
}
conn2 = get(conn, "/api/v1/stats/aggregate", Map.put(query_params, "with_imported", "true"))
assert json_response(conn2, 200)["results"] == %{
"pageviews" => %{"value" => 2}
}
end
test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors,
visits: 2,
bounces: 1,
visit_duration: 200,
pageviews: 10,
date: ~D[2023-01-01]
),
build(:imported_visitors,
visits: 4,
bounces: 1,
visit_duration: 100,
pageviews: 8,
date: ~D[2023-01-02]
),
build(:pageview, timestamp: ~N[2023-01-02 00:10:00])
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "custom",
"date" => "2023-01-02,2023-01-02",
"metrics" => "visitors,visits,pageviews,views_per_visit,bounce_rate,visit_duration",
"compare" => "previous_period",
"with_imported" => "true"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 2, "change" => 100},
"visits" => %{"value" => 5, "change" => 150},
"pageviews" => %{"value" => 9, "change" => -10},
"bounce_rate" => %{"value" => 40, "change" => -20},
"views_per_visit" => %{"value" => 1.0, "change" => 100},
"visit_duration" => %{"value" => 20, "change" => -80}
}
end
test "ignores imported data when filters are applied", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2023-01-01]),
build(:imported_sources, date: ~D[2023-01-01]),
build(:pageview, referrer_source: "Google", timestamp: ~N[2023-01-02 00:10:00])
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2023-01-02",
"metrics" => "visitors",
"filters" => "visit:source==Google",
"compare" => "previous_period",
"with_imported" => "true"
})
assert json_response(conn, 200)["results"] == %{
"visitors" => %{"value" => 1, "change" => 100}
}
end
test "events metric with imported data is disallowed", %{conn: conn, site: site} do
populate_stats(site, [
build(:imported_visitors, date: ~D[2023-01-01])
])
conn =
get(conn, "/api/v1/stats/aggregate", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2023-01-02",
"metrics" => "events",
"with_imported" => "true"
})
assert %{"error" => msg} = json_response(conn, 400)
assert msg == "Metric `events` cannot be queried with imported data"
end
end
describe "filters" do
test "can filter by source", %{conn: conn, site: site} do
populate_stats(site, [

Loading…
Cancel
Save