From c32779a3e590f2da8fe8ee75d458df8abcd3a873 Mon Sep 17 00:00:00 2001 From: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:58:00 +0000 Subject: [PATCH] Timeseries for conversion rate (#3919) * add conversion rate to Stats API timeseries * make sure CR can be queried as the only metric * add a test asserting zeros are returned * add tests for filtering by other properties at the same time * Remove unnecessary validation of params 1. It doesn't make to validate `interval` (and its granularity) in all endpoints. It's only relevant for the main graph. 2. The plug (renamed to `date_validation_plug`) already makes sure that the dates are validated. No need to call the same function again in Top Stats and Funnel endpoints. * add metric validation to main graph * Add tests for main graph API * put conversion rate on the graph * update changelog * Add revenue metrics into metrics.ex * make fn private * avoid setting graph metric to visitors in goal-filtered view --- CHANGELOG.md | 1 + assets/js/dashboard/stats/graph/graph-util.js | 3 + .../js/dashboard/stats/graph/visitor-graph.js | 2 +- lib/plausible/stats/metrics.ex | 27 +++ lib/plausible/stats/timeseries.ex | 37 +++- .../api/external_stats_controller.ex | 18 +- .../controllers/api/stats_controller.ex | 103 +++++----- .../timeseries_test.exs | 178 ++++++++++++++++++ .../api/stats_controller/main_graph_test.exs | 75 ++++++++ 9 files changed, 374 insertions(+), 70 deletions(-) create mode 100644 lib/plausible/stats/metrics.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 43f817873..a4373c74d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ### Added +- Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `time_on_page` metric into the Stats API - County Block List in Site Settings - Query the `views_per_visit` metric based on imported data as well if possible diff --git a/assets/js/dashboard/stats/graph/graph-util.js b/assets/js/dashboard/stats/graph/graph-util.js index bd2126671..dea4848e7 100644 --- a/assets/js/dashboard/stats/graph/graph-util.js +++ b/assets/js/dashboard/stats/graph/graph-util.js @@ -10,6 +10,7 @@ export const METRIC_MAPPING = { 'Total visits': 'visits', 'Bounce rate': 'bounce_rate', 'Unique conversions': 'conversions', + 'Conversion rate': 'conversion_rate', 'Average revenue': 'average_revenue', 'Total revenue': 'total_revenue', } @@ -22,6 +23,7 @@ export const METRIC_LABELS = { 'bounce_rate': 'Bounce Rate', 'visit_duration': 'Visit Duration', 'conversions': 'Converted Visitors', + 'conversion_rate': 'Conversion Rate', 'average_revenue': 'Average Revenue', 'total_revenue': 'Total Revenue', } @@ -34,6 +36,7 @@ export const METRIC_FORMATTER = { 'bounce_rate': (number) => (`${number}%`), 'visit_duration': durationFormatter, 'conversions': numberFormatter, + 'conversion_rate': (number) => (`${number}%`), 'total_revenue': numberFormatter, 'average_revenue': numberFormatter, } diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.js index b7e076d99..fa5df1dd3 100644 --- a/assets/js/dashboard/stats/graph/visitor-graph.js +++ b/assets/js/dashboard/stats/graph/visitor-graph.js @@ -436,7 +436,7 @@ export default class VisitorGraph extends React.Component { const selectableMetrics = topStatData && topStatData.top_stats.map(({ name }) => METRIC_MAPPING[name]).filter(name => name) const canSelectSavedMetric = selectableMetrics && selectableMetrics.includes(savedMetric) - if (query.filters.goal) { + if (query.filters.goal && savedMetric !== 'conversion_rate') { this.setState({ metric: 'conversions' }) } else if (canSelectSavedMetric) { this.setState({ metric: savedMetric }) diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex new file mode 100644 index 000000000..b1c34f2bb --- /dev/null +++ b/lib/plausible/stats/metrics.ex @@ -0,0 +1,27 @@ +defmodule Plausible.Stats.Metrics do + @moduledoc """ + A module listing all available metrics in Plausible. + + Useful for an explicit string to atom conversion. + """ + + use Plausible + + @all_metrics [ + :visitors, + :visits, + :pageviews, + :views_per_visit, + :bounce_rate, + :visit_duration, + :events, + :conversion_rate, + :time_on_page + ] ++ on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) + + @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) + + def from_string!(str) do + Map.fetch!(@metric_mappings, str) + end +end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 053625a90..0306da953 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -3,6 +3,7 @@ defmodule Plausible.Stats.Timeseries do use Plausible alias Plausible.Stats.{Query, Util} import Plausible.Stats.{Base} + import Ecto.Query use Plausible.Stats.Fragments @typep metric :: @@ -18,7 +19,7 @@ defmodule Plausible.Stats.Timeseries do @revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - @event_metrics [:visitors, :pageviews, :events] ++ @revenue_metrics + @event_metrics [:visitors, :pageviews, :events, :conversion_rate] ++ @revenue_metrics @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit] def timeseries(site, query, metrics) do steps = buckets(query) @@ -48,13 +49,17 @@ defmodule Plausible.Stats.Timeseries do |> Map.update!(:date, &date_format/1) |> cast_revenue_metrics_to_money(currency) end) + |> Util.keep_requested_metrics(metrics) end defp events_timeseries(_, _, []), do: [] defp events_timeseries(site, query, metrics) do + metrics = Util.maybe_add_visitors_metric(metrics) + from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) |> select_bucket(site, query) + |> maybe_add_timeseries_conversion_rate(site, query, metrics) |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end @@ -234,6 +239,7 @@ defmodule Plausible.Stats.Timeseries do :visitors -> Map.merge(row, %{visitors: 0}) :visits -> Map.merge(row, %{visits: 0}) :views_per_visit -> Map.merge(row, %{views_per_visit: 0.0}) + :conversion_rate -> Map.merge(row, %{conversion_rate: 0.0}) :bounce_rate -> Map.merge(row, %{bounce_rate: nil}) :visit_duration -> Map.merge(row, %{visit_duration: nil}) :average_revenue -> Map.merge(row, %{average_revenue: nil}) @@ -249,4 +255,33 @@ defmodule Plausible.Stats.Timeseries do else defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results end + + defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do + if :conversion_rate in metrics do + totals_query = query |> Query.remove_event_filters([:goal, :props]) + + totals_timeseries_q = + from(e in base_event_query(site, totals_query), + select: ^select_event_metrics([:visitors]) + ) + |> select_bucket(site, query) + + from(e in subquery(q), + left_join: c in subquery(totals_timeseries_q), + on: e.date == c.date, + select_merge: %{ + total_visitors: c.visitors, + conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + c.visitors, + e.visitors, + c.visitors + ) + } + ) + else + q + end + end end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 1825dbbd0..341026e23 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -2,21 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do use PlausibleWeb, :controller use Plausible.Repo use PlausibleWeb.Plugs.ErrorHandler - alias Plausible.Stats.{Query, Compare, Comparisons} - - @metrics [ - :visitors, - :visits, - :pageviews, - :views_per_visit, - :bounce_rate, - :visit_duration, - :events, - :conversion_rate, - :time_on_page - ] - - @metric_mappings Enum.into(@metrics, %{}, fn metric -> {to_string(metric), metric} end) + alias Plausible.Stats.{Query, Compare, Comparisons, Metrics} def realtime_visitors(conn, _params) do site = conn.assigns.site @@ -117,7 +103,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do {:error, reason} metrics -> - {:ok, Enum.map(metrics, &Map.fetch!(@metric_mappings, &1))} + {:ok, Enum.map(metrics, &Metrics.from_string!/1)} end end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index acc5a56e8..fafb4eda6 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -13,7 +13,7 @@ defmodule PlausibleWeb.Api.StatsController do @revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - plug(:validate_common_input) + plug(:date_validation_plug) @doc """ Returns a time-series based on given parameters. @@ -100,16 +100,11 @@ defmodule PlausibleWeb.Api.StatsController do def main_graph(conn, params) do site = conn.assigns[:site] - with :ok <- validate_params(site, params) do - query = Query.from(site, params) - - selected_metric = - if !params["metric"] || params["metric"] == "conversions" do - :visitors - else - String.to_existing_atom(params["metric"]) - end - + with {:ok, dates} <- parse_date_params(params), + :ok <- validate_interval(params), + :ok <- validate_interval_granularity(site, params, dates), + query = Query.from(site, params), + {:ok, metric} <- parse_and_validate_graph_metric(params, query) do timeseries_query = if query.period == "realtime" do %Query{query | period: "30m"} @@ -117,14 +112,14 @@ defmodule PlausibleWeb.Api.StatsController do query end - timeseries_result = Stats.timeseries(site, timeseries_query, [selected_metric]) + timeseries_result = Stats.timeseries(site, timeseries_query, [metric]) comparison_opts = parse_comparison_opts(params) {comparison_query, comparison_result} = case Comparisons.compare(site, query, params["comparison"], comparison_opts) do {:ok, comparison_query} -> - {comparison_query, Stats.timeseries(site, comparison_query, [selected_metric])} + {comparison_query, Stats.timeseries(site, comparison_query, [metric])} {:error, :not_supported} -> {nil, nil} @@ -137,9 +132,9 @@ defmodule PlausibleWeb.Api.StatsController do site_import = Plausible.Imported.get_earliest_import(site) json(conn, %{ - plot: plot_timeseries(timeseries_result, selected_metric), + plot: plot_timeseries(timeseries_result, metric), labels: labels, - comparison_plot: comparison_result && plot_timeseries(comparison_result, selected_metric), + comparison_plot: comparison_result && plot_timeseries(comparison_result, metric), comparison_labels: comparison_result && label_timeseries(comparison_result, nil), present_index: present_index, interval: query.interval, @@ -207,35 +202,31 @@ defmodule PlausibleWeb.Api.StatsController do def top_stats(conn, params) do site = conn.assigns[:site] - with :ok <- validate_params(site, params) do - query = Query.from(site, params) - - comparison_opts = parse_comparison_opts(params) - - comparison_query = - case Stats.Comparisons.compare(site, query, params["comparison"], comparison_opts) do - {:ok, query} -> query - {:error, _cause} -> nil - end + query = Query.from(site, params) - {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query) + comparison_opts = parse_comparison_opts(params) - site_import = Plausible.Imported.get_earliest_import(site) + comparison_query = + case Stats.Comparisons.compare(site, query, params["comparison"], comparison_opts) do + {:ok, query} -> query + {:error, _cause} -> nil + end - json(conn, %{ - top_stats: top_stats, - interval: query.interval, - sample_percent: sample_percent, - with_imported: with_imported?(query, comparison_query), - imported_source: site_import && SiteImport.label(site_import), - comparing_from: comparison_query && comparison_query.date_range.first, - comparing_to: comparison_query && comparison_query.date_range.last, - from: query.date_range.first, - to: query.date_range.last - }) - else - {:error, message} when is_binary(message) -> bad_request(conn, message) - end + {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query) + + site_import = Plausible.Imported.get_earliest_import(site) + + json(conn, %{ + top_stats: top_stats, + interval: query.interval, + sample_percent: sample_percent, + with_imported: with_imported?(query, comparison_query), + imported_source: site_import && SiteImport.label(site_import), + comparing_from: comparison_query && comparison_query.date_range.first, + comparing_to: comparison_query && comparison_query.date_range.last, + from: query.date_range.first, + to: query.date_range.last + }) end defp present_index_for(site, query, dates) do @@ -464,7 +455,6 @@ defmodule PlausibleWeb.Api.StatsController do site = Plausible.Repo.preload(conn.assigns.site, :owner) with :ok <- Plausible.Billing.Feature.Funnels.check_availability(site.owner), - :ok <- validate_params(site, params), query <- Query.from(site, params), :ok <- validate_funnel_query(query), {funnel_id, ""} <- Integer.parse(funnel_id), @@ -1251,20 +1241,14 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp validate_common_input(conn, _opts) do - case validate_params(conn.assigns[:site], conn.params) do - :ok -> conn + defp date_validation_plug(conn, _opts) do + case parse_date_params(conn.params) do + {:ok, _dates} -> conn {:error, message} when is_binary(message) -> bad_request(conn, message) end end - defp validate_params(site, params) do - with {:ok, dates} <- validate_dates(params), - :ok <- validate_interval(params), - do: validate_interval_granularity(site, params, dates) - end - - defp validate_dates(params) do + defp parse_date_params(params) do params |> Map.take(["from", "to", "date"]) |> Enum.reduce_while({:ok, %{}}, fn {key, value}, {:ok, acc} -> @@ -1321,6 +1305,21 @@ defmodule PlausibleWeb.Api.StatsController do end end + defp parse_and_validate_graph_metric(params, query) do + metric = + case params["metric"] do + nil -> :visitors + "conversions" -> :visitors + m -> Plausible.Stats.Metrics.from_string!(m) + end + + if metric == :conversion_rate and !query.filters["event:goal"] do + {:error, "Metric `:conversion_rate` can only be queried with a goal filter"} + else + {:ok, metric} + end + end + defp bad_request(conn, message, extra \\ %{}) do payload = Map.merge(extra, %{error: message}) diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs index 78c203a18..528d3584a 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs @@ -1091,6 +1091,184 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do end describe "metrics" do + test "returns conversion rate as 0 when no stats exist", %{ + conn: conn, + site: site + } do + insert(:goal, site: site, event_name: "Signup") + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup", + "period" => "7d", + "date" => "2021-01-10" + }) + + Enum.each(json_response(conn, 200)["results"], fn bucket -> + bucket["conversion_rate"] == 0.0 + end) + end + + test "returns conversion rate when goal filter is applied", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, name: "Signup", timestamp: ~N[2021-01-04 00:00:00]), + build(:event, name: "Signup", timestamp: ~N[2021-01-04 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-04 00:00:00]), + build(:event, name: "Signup", timestamp: ~N[2021-01-05 00:00:00]) + ]) + + insert(:goal, site: site, event_name: "Signup") + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup", + "period" => "7d", + "date" => "2021-01-10" + }) + + assert [first, second | _] = json_response(conn, 200)["results"] + + assert [first, second] == [ + %{ + "date" => "2021-01-04", + "conversion_rate" => 66.7 + }, + %{ + "date" => "2021-01-05", + "conversion_rate" => 100.0 + } + ] + end + + test "returns conversion rate with a goal + custom prop filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Signup", + "meta.key": ["author"], + "meta.value": ["Teet"], + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, + name: "Signup", + "meta.key": ["author"], + "meta.value": ["Tiit"], + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, name: "Signup", timestamp: ~N[2021-01-04 00:12:00]), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Teet"], + timestamp: ~N[2021-01-04 00:12:00] + ) + ]) + + insert(:goal, site: site, event_name: "Signup") + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup;event:props:author==Teet", + "period" => "7d", + "date" => "2021-01-10" + }) + + [first | _] = json_response(conn, 200)["results"] + + assert first == %{ + "date" => "2021-01-04", + "conversion_rate" => 25.0 + } + end + + test "returns conversion rate with a goal + page filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Signup", + pathname: "/yes", + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, + name: "Signup", + pathname: "/no", + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, name: "Signup", timestamp: ~N[2021-01-04 00:12:00]), + build(:pageview, pathname: "/yes", timestamp: ~N[2021-01-04 00:12:00]), + build(:pageview, pathname: "/yes", timestamp: ~N[2021-01-04 00:12:00]) + ]) + + insert(:goal, site: site, event_name: "Signup") + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup;event:page==/yes", + "period" => "7d", + "date" => "2021-01-10" + }) + + [first | _] = json_response(conn, 200)["results"] + + assert first == %{ + "date" => "2021-01-04", + "conversion_rate" => 33.3 + } + end + + test "returns conversion rate with a goal + session filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Signup", + screen_size: "Mobile", + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, + name: "Signup", + screen_size: "Laptop", + timestamp: ~N[2021-01-04 00:12:00] + ), + build(:event, name: "Signup", timestamp: ~N[2021-01-04 00:12:00]), + build(:pageview, screen_size: "Mobile", timestamp: ~N[2021-01-04 00:12:00]), + build(:pageview, screen_size: "Mobile", timestamp: ~N[2021-01-04 00:12:00]) + ]) + + insert(:goal, site: site, event_name: "Signup") + + conn = + get(conn, "/api/v1/stats/timeseries", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup;visit:device==Mobile", + "period" => "7d", + "date" => "2021-01-10" + }) + + [first | _] = json_response(conn, 200)["results"] + + assert first == %{ + "date" => "2021-01-04", + "conversion_rate" => 33.3 + } + end + test "validates that conversion_rate cannot be queried without a goal filter", %{ conn: conn, site: site diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 2174943ef..d44662d77 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -433,6 +433,51 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end end + describe "GET /api/stats/main-graph - conversion_rate plot" do + setup [:create_user, :log_in, :create_new_site] + + test "returns 400 when conversion rate is queried without a goal filter", %{ + conn: conn, + site: site + } do + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=conversion_rate" + ) + + assert %{"error" => error} = json_response(conn, 400) + assert error =~ "can only be queried with a goal filter" + end + + test "displays conversion_rate for a month", %{conn: conn, site: site} do + insert(:goal, site: site, event_name: "Signup") + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:event, name: "Signup", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00]), + build(:event, name: "Signup", timestamp: ~N[2021-01-31 00:00:00]) + ]) + + filters = Jason.encode!(%{goal: "Signup"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=conversion_rate&filters=#{filters}" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + assert Enum.count(plot) == 31 + + assert List.first(plot) == 33.3 + assert Enum.at(plot, 10) == 0.0 + assert List.last(plot) == 50.0 + end + end + describe "GET /api/stats/main-graph - bounce_rate plot" do setup [:create_user, :log_in, :create_new_site, :add_imported_data] @@ -969,6 +1014,36 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert 4 == Enum.sum(plot) assert 0 == Enum.sum(comparison_plot) end + + test "plots conversion rate previous period comparison", %{site: site, conn: conn} do + insert(:goal, site: site, event_name: "Signup") + + populate_stats(site, [ + build(:event, name: "Signup", timestamp: ~N[2021-01-01 00:01:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:01:00]), + build(:pageview, timestamp: ~N[2021-01-01 00:01:00]), + build(:event, name: "Signup", timestamp: ~N[2021-01-08 00:01:00]), + build(:pageview, timestamp: ~N[2021-01-08 00:01:00]) + ]) + + filters = Jason.encode!(%{goal: "Signup"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=7d&date=2021-01-14&comparison=previous_period&metric=conversion_rate&filters=#{filters}" + ) + + assert %{ + "plot" => this_week_plot, + "comparison_plot" => last_week_plot, + "imported_source" => "Google Analytics", + "with_imported" => false + } = json_response(conn, 200) + + assert this_week_plot == [50.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + assert last_week_plot == [33.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + end end @tag :full_build_only