diff --git a/CHANGELOG.md b/CHANGELOG.md index 2791742a1..f3b37ff6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## Unreleased ### Added +- Add support for `with_imported=true` in Stats API aggregate endpoint - Ability to use '--' instead of '=' sign in the `tagged-events` classnames - 'Last updated X seconds ago' info to 'current visitors' tooltips - Add support for more Bamboo adapters, i.e. `Bamboo.MailgunAdapter`, `Bamboo.MandrillAdapter`, `Bamboo.SendGridAdapter` plausible/analytics#2649 diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 1c0dcc775..d628d7612 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Aggregate do alias Plausible.Stats.Query use Plausible.ClickhouseRepo - import Plausible.Stats.{Base, Imported} + import Plausible.Stats.{Base, Imported, Util} @event_metrics [:visitors, :pageviews, :events, :sample_percent] @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent] @@ -43,6 +43,7 @@ defmodule Plausible.Stats.Aggregate do |> select_session_metrics(metrics, query) |> merge_imported(site, query, :aggregate, metrics) |> ClickhouseRepo.one() + |> remove_internal_visits_metric() end defp aggregate_time_on_page(site, query) do diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex index 4e50d408b..cc15ecd47 100644 --- a/lib/plausible/stats/comparisons.ex +++ b/lib/plausible/stats/comparisons.ex @@ -53,20 +53,16 @@ defmodule Plausible.Stats.Comparisons do it will be shifted to start on Sunday, January 2nd, 2022 instead of January 1st. Defaults to false. - * `:include_imported?` - includes imported data in the generated comparison - query if there is imported data. Defaults to true. - """ def compare(%Plausible.Site{} = site, %Stats.Query{} = source_query, mode, opts \\ []) do opts = opts |> Keyword.put_new(:now, Timex.now(site.timezone)) |> Keyword.put_new(:match_day_of_week?, false) - |> Keyword.put_new(:include_imported?, true) with :ok <- validate_mode(source_query, mode), {:ok, comparison_query} <- do_compare(source_query, mode, opts), - comparison_query <- maybe_include_imported(comparison_query, site, opts), + comparison_query <- maybe_include_imported(comparison_query, source_query, site), do: {:ok, comparison_query} end @@ -165,8 +161,9 @@ defmodule Plausible.Stats.Comparisons do Date.add(date, -days_to_subtract) end - defp maybe_include_imported(query, site, opts) do - include_imported? = Stats.Query.include_imported?(query, site, opts[:include_imported?]) + defp maybe_include_imported(query, source_query, site) do + requested? = source_query.imported_data_requested + include_imported? = Stats.Query.include_imported?(query, site, requested?) %Stats.Query{query | include_imported: include_imported?} end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 12221a126..4dcb6401a 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -4,6 +4,7 @@ defmodule Plausible.Stats.Query do period: nil, filters: %{}, sample_threshold: 20_000_000, + imported_data_requested: false, include_imported: false @default_sample_threshold 20_000_000 @@ -20,8 +21,7 @@ defmodule Plausible.Stats.Query do interval: params["interval"] || Interval.default_for_period(params["period"]), date_range: Date.range(date, date), filters: FilterParser.parse_filters(params["filters"]), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - include_imported: false + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } end @@ -35,7 +35,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "7d"} = params) do @@ -49,7 +49,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "30d"} = params) do @@ -63,7 +63,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "month"} = params) do @@ -79,7 +79,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "6mo"} = params) do @@ -98,7 +98,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "12mo"} = params) do @@ -117,7 +117,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "year"} = params) do @@ -134,7 +134,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(site, %{"period" => "all"} = params) do @@ -194,7 +194,7 @@ defmodule Plausible.Stats.Query do filters: FilterParser.parse_filters(params["filters"]), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } - |> maybe_include_imported(site, params) + |> put_imported_opts(site, params) end def from(tz, params) do @@ -248,9 +248,12 @@ defmodule Plausible.Stats.Query do end end - defp maybe_include_imported(query, site, params) do + defp put_imported_opts(query, site, params) do requested? = params["with_imported"] == "true" - %{query | include_imported: include_imported?(query, site, requested?)} + + query + |> Map.put(:imported_data_requested, requested?) + |> Map.put(:include_imported, include_imported?(query, site, requested?)) end @spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean() diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 71a82d4f9..046550660 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -8,12 +8,16 @@ defmodule Plausible.Stats.Util do is needed to calculate these from imported data. This function removes that metric from all entries in the results list. """ - def remove_internal_visits_metric(results, metrics) do + def remove_internal_visits_metric(results, metrics) when is_list(results) do if :bounce_rate in metrics or :visit_duration in metrics do results - |> Enum.map(&Map.delete(&1, :__internal_visits)) + |> Enum.map(&remove_internal_visits_metric/1) else results end end + + def remove_internal_visits_metric(result) when is_map(result) do + Map.delete(result, :__internal_visits) + end end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 6256c057e..227543fc6 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -42,10 +42,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController do Plausible.Stats.aggregate(site, query, metrics) end - results = - results - |> Map.take(metrics) - json(conn, %{results: results}) else {:error, msg} -> @@ -141,6 +137,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end) end + defp validate_metric("events", nil, %{include_imported: true}) do + {:error, "Metric `events` cannot be queried with imported data"} + end + defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric} defp validate_metric(metric, property, query) when metric in @session_metrics do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 93d63ff82..bba59a5f2 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -1304,8 +1304,7 @@ defmodule PlausibleWeb.Api.StatsController do [ from: params["compare_from"], to: params["compare_to"], - match_day_of_week?: params["match_day_of_week"] == "true", - include_imported?: params["with_imported"] == "true" + match_day_of_week?: params["match_day_of_week"] == "true" ] end diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs index 657c1894f..c0a8e8d0e 100644 --- a/test/plausible/stats/comparisons_test.exs +++ b/test/plausible/stats/comparisons_test.exs @@ -1,6 +1,7 @@ defmodule Plausible.Stats.ComparisonsTest do use Plausible.DataCase alias Plausible.Stats.{Query, Comparisons} + import Plausible.TestUtils describe "with period set to this month" do test "shifts back this month period when mode is previous_period" do @@ -200,4 +201,16 @@ defmodule Plausible.Stats.ComparisonsTest do Comparisons.compare(site, query, "custom", from: "2022-05-30", to: "2022-05-25") end end + + describe "include_imported" do + setup [:create_user, :create_new_site, :add_imported_data] + + test "defaults to source_query.include_imported", %{site: site} do + query = Query.from(site, %{"period" => "day", "date" => "2023-01-01"}) + assert query.include_imported == false + + {:ok, comparison_query} = Comparisons.compare(site, query, "previous_period") + assert comparison_query.include_imported == false + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 2236e324a..8f7ff2ec7 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -1,5 +1,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do use PlausibleWeb.ConnCase + import Plausible.TestUtils setup [:create_user, :create_new_site, :create_api_key, :use_api_key] @user_id 123 @@ -250,6 +251,116 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do end end + describe "with imported data" do + setup :add_imported_data + + test "does not count imported stats unless specified", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2023-01-01]), + build(:pageview, timestamp: ~N[2023-01-01 00:00:00]) + ]) + + query_params = %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2023-01-01", + "metrics" => "pageviews" + } + + conn1 = get(conn, "/api/v1/stats/aggregate", query_params) + + assert json_response(conn1, 200)["results"] == %{ + "pageviews" => %{"value" => 1} + } + + conn2 = get(conn, "/api/v1/stats/aggregate", Map.put(query_params, "with_imported", "true")) + + assert json_response(conn2, 200)["results"] == %{ + "pageviews" => %{"value" => 2} + } + end + + test "counts imported stats when comparing with previous period", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, + visits: 2, + bounces: 1, + visit_duration: 200, + pageviews: 10, + date: ~D[2023-01-01] + ), + build(:imported_visitors, + visits: 4, + bounces: 1, + visit_duration: 100, + pageviews: 8, + date: ~D[2023-01-02] + ), + build(:pageview, timestamp: ~N[2023-01-02 00:10:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "custom", + "date" => "2023-01-02,2023-01-02", + "metrics" => "visitors,visits,pageviews,views_per_visit,bounce_rate,visit_duration", + "compare" => "previous_period", + "with_imported" => "true" + }) + + assert json_response(conn, 200)["results"] == %{ + "visitors" => %{"value" => 2, "change" => 100}, + "visits" => %{"value" => 5, "change" => 150}, + "pageviews" => %{"value" => 9, "change" => -10}, + "bounce_rate" => %{"value" => 40, "change" => -20}, + "views_per_visit" => %{"value" => 1.0, "change" => 100}, + "visit_duration" => %{"value" => 20, "change" => -80} + } + end + + test "ignores imported data when filters are applied", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2023-01-01]), + build(:imported_sources, date: ~D[2023-01-01]), + build(:pageview, referrer_source: "Google", timestamp: ~N[2023-01-02 00:10:00]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2023-01-02", + "metrics" => "visitors", + "filters" => "visit:source==Google", + "compare" => "previous_period", + "with_imported" => "true" + }) + + assert json_response(conn, 200)["results"] == %{ + "visitors" => %{"value" => 1, "change" => 100} + } + end + + test "events metric with imported data is disallowed", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2023-01-01]) + ]) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2023-01-02", + "metrics" => "events", + "with_imported" => "true" + }) + + assert %{"error" => msg} = json_response(conn, 400) + assert msg == "Metric `events` cannot be queried with imported data" + end + end + describe "filters" do test "can filter by source", %{conn: conn, site: site} do populate_stats(site, [