From 1909743b908980fc22bce4dc17dde88f7529febc Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 21 Mar 2024 09:38:44 +0200 Subject: [PATCH] Move conversion_rate logic from elixir to clickhouse (#3887) * Separate out query building from pagination/execution logic. * Refactor pageview_goals breakdown query, removing index column from results * Remove zip_columns logic * Use common pagination util * Do everything in a single query for breakdowns for goals * Order in DB * Make sure column order is identical * Calculate CR within the goal breakdown query * Calculate CR for property breakdowns * WIP: Calculate group CR * CR with order_by * Compatibility fix * Import Ecto.Query and cleanup * handle total_visitors the same way as add_percentage * Handle conversion_rate in aggregate.ex * Solve rebase fail * Simplify maybe_add_group_conversion_rate * Add conversion_rate defaults to 0 test * Add test for conversion_rate should not be calculated with imported data (failing here and on master) * Dont include imported data when breakdown by prop or goal * Remove revenue_nils --- extra/lib/plausible/stats/goal/revenue.ex | 20 +- lib/plausible/stats/aggregate.ex | 36 +- lib/plausible/stats/base.ex | 251 +++++++------- lib/plausible/stats/breakdown.ex | 314 ++++++++---------- lib/plausible/stats/timeseries.ex | 6 +- lib/plausible/stats/util.ex | 11 +- .../aggregate_test.exs | 16 + .../api/stats_controller/conversions_test.exs | 36 ++ 8 files changed, 346 insertions(+), 344 deletions(-) diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index 46c4049b9..7b6e3368f 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -10,21 +10,17 @@ defmodule Plausible.Stats.Goal.Revenue do @revenue_metrics end - def total_revenue_query(query) do - from(e in query, - select_merge: %{ - total_revenue: - fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) - } + def total_revenue_query() do + dynamic( + [e], + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) ) end - def average_revenue_query(query) do - from(e in query, - select_merge: %{ - average_revenue: - fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) - } + def average_revenue_query() do + dynamic( + [e], + fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) ) end diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 56ded33f8..68b96183a 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -11,7 +11,9 @@ defmodule Plausible.Stats.Aggregate do :visitors, :pageviews, :events, - :sample_percent + :sample_percent, + :conversion_rate, + :total_visitors ] ++ @revenue_metrics @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent] @@ -45,7 +47,6 @@ defmodule Plausible.Stats.Aggregate do Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task]) |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end) - |> maybe_put_cr(site, query, metrics) |> Util.keep_requested_metrics(metrics) |> cast_revenue_metrics_to_money(currency) |> Enum.map(&maybe_round_value/1) @@ -53,45 +54,20 @@ defmodule Plausible.Stats.Aggregate do |> Enum.into(%{}) end - defp maybe_put_cr(aggregate_result, site, query, metrics) do - if :conversion_rate in metrics do - all = - query - |> Query.remove_event_filters([:goal, :props]) - |> then(fn query -> aggregate_events(site, query, [:visitors]) end) - |> Map.fetch!(:visitors) - - converted = aggregate_result.visitors - - cr = Util.calculate_cr(all, converted) - - aggregate_result = Map.put(aggregate_result, :conversion_rate, cr) - - if :total_visitors in metrics do - Map.put(aggregate_result, :total_visitors, all) - else - aggregate_result - end - else - aggregate_result - end - end - defp aggregate_events(_, _, []), do: %{} defp aggregate_events(site, query, metrics) do - from(e in base_event_query(site, query), select: %{}) - |> select_event_metrics(metrics) + from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) |> merge_imported(site, query, :aggregate, metrics) + |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported) |> ClickhouseRepo.one() end defp aggregate_sessions(_, _, []), do: %{} defp aggregate_sessions(site, query, metrics) do - from(e in query_sessions(site, query), select: %{}) + from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) |> filter_converted_sessions(site, query) - |> select_session_metrics(metrics, query) |> merge_imported(site, query, :aggregate, metrics) |> ClickhouseRepo.one() |> Util.keep_requested_metrics(metrics) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 83b4ad6d2..463461bf8 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -209,163 +209,152 @@ defmodule Plausible.Stats.Base do def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q - def select_event_metrics(q, []), do: q + def select_event_metrics(metrics) do + metrics + |> Enum.map(&select_event_metric/1) + |> Enum.reduce(%{}, &Map.merge/2) + end - def select_event_metrics(q, [:pageviews | rest]) do - from(e in q, - select_merge: %{ - pageviews: + defp select_event_metric(:pageviews) do + %{ + pageviews: + dynamic( + [e], fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) - } - ) - |> select_event_metrics(rest) + ) + } end - def select_event_metrics(q, [:events | rest]) do - from(e in q, - select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")} - ) - |> select_event_metrics(rest) + defp select_event_metric(:events) do + %{ + events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))")) + } end - def select_event_metrics(q, [:visitors | rest]) do - from(e in q, - select_merge: %{ - visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors) - } - ) - |> select_event_metrics(rest) + defp select_event_metric(:visitors) do + %{ + visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)) + } end on_full_build do - def select_event_metrics(q, [:total_revenue | rest]) do - q - |> Plausible.Stats.Goal.Revenue.total_revenue_query() - |> select_event_metrics(rest) + defp select_event_metric(:total_revenue) do + %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()} end - def select_event_metrics(q, [:average_revenue | rest]) do - q - |> Plausible.Stats.Goal.Revenue.average_revenue_query() - |> select_event_metrics(rest) + defp select_event_metric(:average_revenue) do + %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()} end end - def select_event_metrics(q, [:sample_percent | rest]) do - from(e in q, - select_merge: %{ - sample_percent: + defp select_event_metric(:sample_percent) do + %{ + sample_percent: + dynamic( + [], fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - } - ) - |> select_event_metrics(rest) + ) + } end - def select_event_metrics(q, [:percentage | rest]) do - q |> select_event_metrics(rest) - end + defp select_event_metric(:percentage), do: %{} + defp select_event_metric(:conversion_rate), do: %{} + defp select_event_metric(:total_visitors), do: %{} - def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}") + defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") - def select_session_metrics(q, [], _query), do: q + def select_session_metrics(metrics, query) do + metrics + |> Enum.map(&select_session_metric(&1, query)) + |> Enum.reduce(%{}, &Map.merge/2) + end - def select_session_metrics(q, [:bounce_rate | rest], query) do + defp select_session_metric(:bounce_rate, query) do condition = dynamic_filter_condition(query, "event:page", :entry_page) - from(s in q, - select_merge: - ^%{ - bounce_rate: - dynamic( - [], - fragment( - "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", - ^condition, - ^condition - ) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } - ) - |> select_session_metrics(rest, query) + %{ + bounce_rate: + dynamic( + [], + fragment( + "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", + ^condition, + ^condition + ) + ), + __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) + } end - def select_session_metrics(q, [:visits | rest], query) do - from(s in q, - select_merge: %{ - visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign) - } - ) - |> select_session_metrics(rest, query) + defp select_session_metric(:visits, _query) do + %{ + visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)) + } end - def select_session_metrics(q, [:pageviews | rest], query) do - from(s in q, - select_merge: %{ - pageviews: + defp select_session_metric(:pageviews, _query) do + %{ + pageviews: + dynamic( + [s], fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) - } - ) - |> select_session_metrics(rest, query) + ) + } end - def select_session_metrics(q, [:events | rest], query) do - from(s in q, - select_merge: %{ - events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) - } - ) - |> select_session_metrics(rest, query) + defp select_session_metric(:events, _query) do + %{ + events: + dynamic( + [s], + fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) + ) + } end - def select_session_metrics(q, [:visitors | rest], query) do - from(s in q, - select_merge: %{ - visitors: + defp select_session_metric(:visitors, _query) do + %{ + visitors: + dynamic( + [s], selected_as( fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id), :visitors ) - } - ) - |> select_session_metrics(rest, query) + ) + } end - def select_session_metrics(q, [:visit_duration | rest], query) do - from(s in q, - select_merge: %{ - :visit_duration => - fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), - __internal_visits: fragment("toUInt32(sum(sign))") - } - ) - |> select_session_metrics(rest, query) + defp select_session_metric(:visit_duration, _query) do + %{ + visit_duration: + dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")), + __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) + } end - def select_session_metrics(q, [:views_per_visit | rest], query) do - from(s in q, - select_merge: %{ - views_per_visit: - fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign), - __internal_visits: fragment("toUInt32(sum(sign))") - } - ) - |> select_session_metrics(rest, query) + defp select_session_metric(:views_per_visit, _query) do + %{ + views_per_visit: + dynamic( + [s], + fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign) + ), + __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) + } end - def select_session_metrics(q, [:sample_percent | rest], query) do - from(e in q, - select_merge: %{ - sample_percent: + defp select_session_metric(:sample_percent, _query) do + %{ + sample_percent: + dynamic( + [], fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - } - ) - |> select_session_metrics(rest, query) + ) + } end - def select_session_metrics(q, [:percentage | rest], query) do - q |> select_session_metrics(rest, query) - end + defp select_session_metric(:percentage, _query), do: %{} def dynamic_filter_condition(query, filter_key, db_field) do case query && query.filters && query.filters[filter_key] do @@ -586,7 +575,7 @@ defmodule Plausible.Stats.Base do |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) end - defp total_visitors_subquery(site, %Query{include_imported: true} = query) do + defp total_visitors_subquery(site, query, true) do dynamic( [e], selected_as( @@ -597,14 +586,16 @@ defmodule Plausible.Stats.Base do ) end - defp total_visitors_subquery(site, query) do + defp total_visitors_subquery(site, query, false) do dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) end def add_percentage_metric(q, site, query, metrics) do if :percentage in metrics do q - |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)}) + |> select_merge( + ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)} + ) |> select_merge(%{ percentage: fragment( @@ -618,4 +609,32 @@ defmodule Plausible.Stats.Base do q end end + + # Adds conversion_rate metric to query, calculated as + # X / Y where Y is the same breakdown value without goal or props + # filters. + def maybe_add_conversion_rate(q, site, query, metrics, opts) do + if :conversion_rate in metrics do + include_imported = Keyword.fetch!(opts, :include_imported) + + total_query = query |> Query.remove_event_filters([:goal, :props]) + + # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL + subquery(q) + |> select_merge( + ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)} + ) + |> select_merge([e], %{ + conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + selected_as(:__total_visitors), + e.visitors, + selected_as(:__total_visitors) + ) + }) + else + q + end + end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index ae33e4e73..822c00df1 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -4,6 +4,7 @@ defmodule Plausible.Stats.Breakdown do use Plausible.Stats.Fragments import Plausible.Stats.{Base, Imported} + import Ecto.Query require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Query, Util} @@ -52,50 +53,74 @@ defmodule Plausible.Stats.Breakdown do metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - event_results = + event_q = if Enum.any?(event_goals) do site - |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true) - |> transform_keys(%{name: :goal}) - |> cast_revenue_metrics_to_money(revenue_goals) + |> breakdown_events(event_query, "event:name", metrics_to_select) + |> apply_pagination(pagination) else - [] + nil end - {limit, page} = pagination - offset = (page - 1) * limit - - page_results = + page_q = if Enum.any?(pageview_goals) do page_exprs = Enum.map(pageview_goals, & &1.page_path) page_regexes = Enum.map(page_exprs, &page_regex/1) + select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil + from(e in base_event_query(site, query), order_by: [desc: fragment("uniq(?)", e.user_id)], - limit: ^limit, - offset: ^offset, where: fragment( "notEmpty(multiMatchAllIndices(?, ?) as indices)", e.pathname, ^page_regexes ) and e.name == "pageview", - group_by: fragment("index"), - select: %{ - index: fragment("arrayJoin(indices) as index"), - goal: fragment("concat('Visit ', ?[index])", ^page_exprs) - } + array_join: index in fragment("indices"), + group_by: index, + select: %{} ) - |> select_event_metrics(metrics_to_select -- @revenue_metrics) - |> ClickhouseRepo.all() - |> Enum.map(fn row -> Map.delete(row, :index) end) + |> select_merge(^select_columns) + # :TRICKY: name is added last to make sure both queries add columns in the same order + |> select_merge([_, index], %{ + name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) + }) + |> apply_pagination(pagination) else - [] + nil end - zip_results(event_results, page_results, :goal, metrics_to_select) - |> maybe_add_cr(site, query, nil, metrics) - |> Util.keep_requested_metrics(metrics) + full_q = + case {event_q, page_q} do + {nil, nil} -> + nil + + {event_q, nil} -> + event_q + + {nil, page_q} -> + page_q + + {event_q, page_q} -> + from( + e in subquery(union_all(event_q, ^page_q)), + # :TODO: Handle other orderings + order_by: [desc: e.visitors] + ) + |> apply_pagination(pagination) + end + + if full_q do + full_q + |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) + |> ClickhouseRepo.all() + |> transform_keys(%{name: :goal}) + |> cast_revenue_metrics_to_money(revenue_goals) + |> Util.keep_requested_metrics(metrics) + else + [] + end end def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do @@ -110,11 +135,11 @@ defmodule Plausible.Stats.Breakdown do if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) - breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination) + breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select) + |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) + |> paginate_and_execute(metrics, pagination) + |> transform_keys(%{name: custom_prop}) |> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) - |> sort_results(metrics_to_select) - |> maybe_add_cr(site, query, nil, metrics) - |> Util.keep_requested_metrics(metrics) end def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do @@ -125,10 +150,10 @@ defmodule Plausible.Stats.Breakdown do event_result = site - |> breakdown_events(query, "event:page", event_metrics, pagination) + |> breakdown_events(query, property, event_metrics) + |> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics) + |> paginate_and_execute(metrics, pagination) |> maybe_add_time_on_page(site, query, metrics) - |> maybe_add_cr(site, query, property, metrics, pagination) - |> Util.keep_requested_metrics(metrics) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) @@ -149,7 +174,8 @@ defmodule Plausible.Stats.Breakdown do {limit, _page} = pagination session_result = - breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1}) + breakdown_sessions(site, new_query, "visit:entry_page", session_metrics) + |> paginate_and_execute(session_metrics, {limit, 1}) |> transform_keys(%{entry_page: :page}) metrics = metrics ++ [:page] @@ -166,7 +192,9 @@ defmodule Plausible.Stats.Breakdown do def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) - breakdown_events(site, query, property, metrics, pagination) + + breakdown_events(site, query, property, metrics) + |> paginate_and_execute(metrics, pagination) end def breakdown(site, query, property, metrics, pagination, opts) do @@ -174,9 +202,9 @@ defmodule Plausible.Stats.Breakdown do metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - breakdown_sessions(site, query, property, metrics_to_select, pagination) - |> maybe_add_cr(site, query, property, metrics, pagination) - |> Util.keep_requested_metrics(metrics) + breakdown_sessions(site, query, property, metrics_to_select) + |> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics) + |> paginate_and_execute(metrics, pagination) end defp zip_results(event_result, session_result, property, metrics) do @@ -197,35 +225,31 @@ defmodule Plausible.Stats.Breakdown do |> sort_results(metrics) end - defp breakdown_sessions(_, _, _, [], _), do: [] - - defp breakdown_sessions(site, query, property, metrics, pagination) do + defp breakdown_sessions(site, query, property, metrics) do from(s in query_sessions(site, query), order_by: [desc: fragment("uniq(?)", s.user_id)], - select: %{} + select: ^select_session_metrics(metrics, query) ) |> filter_converted_sessions(site, query) |> do_group_by(property) - |> select_session_metrics(metrics, query) |> merge_imported(site, query, property, metrics) |> add_percentage_metric(site, query, metrics) - |> apply_pagination(pagination) - |> ClickhouseRepo.all() - |> transform_keys(%{operating_system: :os}) - |> Util.keep_requested_metrics(metrics) end - defp breakdown_events(_, _, _, [], _), do: [] - - defp breakdown_events(site, query, property, metrics, pagination) do + defp breakdown_events(site, query, property, metrics) do from(e in base_event_query(site, query), order_by: [desc: fragment("uniq(?)", e.user_id)], - select: %{} + select: ^select_event_metrics(metrics) ) |> do_group_by(property) - |> select_event_metrics(metrics) |> merge_imported(site, query, property, metrics) |> add_percentage_metric(site, query, metrics) + end + + defp paginate_and_execute(_, [], _), do: [] + + defp paginate_and_execute(q, metrics, pagination) do + q |> apply_pagination(pagination) |> ClickhouseRepo.all() |> transform_keys(%{operating_system: :os}) @@ -423,18 +447,18 @@ defmodule Plausible.Stats.Breakdown do from( e in q, select_merge: %{ - ^prop => + name: selected_as( fragment( "if(not empty(?), ?, '(none)')", get_by_key(e, :meta, ^prop), get_by_key(e, :meta, ^prop) ), - :breakdown_prop_value + :name ) }, - group_by: selected_as(:breakdown_prop_value), - order_by: {:asc, selected_as(:breakdown_prop_value)} + group_by: selected_as(:name), + order_by: {:asc, selected_as(:name)} ) end @@ -652,130 +676,26 @@ defmodule Plausible.Stats.Breakdown do ) end - defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do - cond do - :conversion_rate not in metrics -> breakdown_results - Enum.empty?(breakdown_results) -> breakdown_results - is_nil(property) -> add_absolute_cr(breakdown_results, site, query) - true -> add_cr(breakdown_results, site, query, property, metrics, pagination) - end - end - - # This function injects a conversion_rate metric into every - # breakdown result map. It is calculated as X / Y, where: - # - # * X is the number of conversions for a breakdown - # result (conversion = number of visitors who - # completed the filtered goal with the filtered - # custom properties). - # - # * Y is the number of all visitors for this breakdown - # result without the `event:goal` and `event:props:*` - # filters. - defp add_cr(breakdown_results, site, query, property, metrics, pagination) do - property_atom = Plausible.Stats.Filters.without_prefix(property) - - items = - Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end) - - query_without_goal = - query - |> Query.put_filter(property, {:member, items}) - |> Query.remove_event_filters([:goal, :props]) - - # Here, we're always only interested in the first page of results - # - the :member filter makes sure that the results always match with - # the items in the given breakdown_results list - page = 1 - - # For browser/os versions we need to fetch a lot more entries than the - # pagination limit. This is because many entries can correspond to a - # single version number and we need to make sure that the results - # without goal filter will include all those combinations of browsers/os-s - # and their versions that were present in the `breakdown_results` array. - {pagination_limit, find_match_fn} = - case property_atom do - :browser_version -> - pagination_limit = min(elem(pagination, 0) * 10, 3_000) - - find_match_fn = fn total, conversion -> - total[:browser_version] == conversion[:browser_version] && - total[:browser] == conversion[:browser] - end - - {pagination_limit, find_match_fn} - - :os_version -> - pagination_limit = min(elem(pagination, 0) * 5, 3_000) - - find_match_fn = fn total, conversion -> - total[:os_version] == conversion[:os_version] && - total[:os] == conversion[:os] - end - - {pagination_limit, find_match_fn} - - _ -> - {elem(pagination, 0), - fn total, conversion -> - total[property_atom] == conversion[property_atom] - end} - end - - pagination = {pagination_limit, page} - - res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination) + defp group_by_field_names("event:props:" <> _prop), do: [:name] + defp group_by_field_names("visit:os"), do: [:operating_system] + defp group_by_field_names("visit:os_version"), do: [:os, :os_version] + defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version] - Enum.map(breakdown_results, fn item -> - without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item)) + defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)] - {conversion_rate, total_visitors} = - if without_goal do - {Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors} - else - Sentry.capture_message( - "Unable to find a conversion_rate divisor from a breakdown response", - extra: %{ - domain: site.domain, - query: inspect(query), - property: property, - pagination: inspect(pagination), - item_not_found: inspect(item) - } - ) - - {"N/A", "N/A"} - end - - if :total_visitors in metrics do - item - |> Map.put(:conversion_rate, conversion_rate) - |> Map.put(:total_visitors, total_visitors) - else - Map.put(item, :conversion_rate, conversion_rate) - end - end) + defp on_matches_group_by(fields) do + Enum.reduce(fields, nil, &fields_equal/2) end - # Similar to `add_cr/5`, injects a conversion_rate metric into - # every breakdown result. However, a single divisor is used in - # the CR calculation across all breakdown results. That is the - # number of visitors without `event:goal` and `event:props:*` - # filters. - # - # This is useful when we're only interested in the conversions - # themselves - not how well a certain property such as browser - # or page converted. - defp add_absolute_cr(breakdown_results, site, query) do - total_q = Query.remove_event_filters(query, [:goal, :props]) + defp outer_order_by(fields) do + Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end) + end - %{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors]) + defp fields_equal(field_name, nil), + do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name)) - breakdown_results - |> Enum.map(fn goal -> - Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors])) - end) - end + defp fields_equal(field_name, condition), + do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition) defp sort_results(results, metrics) do Enum.sort_by( @@ -790,6 +710,54 @@ defmodule Plausible.Stats.Breakdown do ) end + # This function injects a conversion_rate metric into + # a breakdown query. It is calculated as X / Y, where: + # + # * X is the number of conversions for a breakdown + # result (conversion = number of visitors who + # completed the filtered goal with the filtered + # custom properties). + # + # * Y is the number of all visitors for this breakdown + # result without the `event:goal` and `event:props:*` + # filters. + defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do + if :conversion_rate in metrics do + breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props]) + + breakdown_total_visitors_q = + breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors]) + + from(e in subquery(q), + left_join: c in subquery(breakdown_total_visitors_q), + on: ^on_matches_group_by(group_by_field_names(property)), + select_merge: %{ + total_visitors: c.visitors, + conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), 0)", + c.visitors, + e.visitors, + c.visitors + ) + }, + order_by: [desc: e.visitors], + order_by: ^outer_order_by(group_by_field_names(property)) + ) + else + q + end + end + + # When querying custom event goals and pageviewgoals together, UNION ALL is used + # so the same fields must be present on both sides of the union. This change to the + # query will ensure that we don't unnecessarily read revenue column for pageview goals + defp mark_revenue_as_nil(select_columns) do + select_columns + |> Map.replace(:total_revenue, nil) + |> Map.replace(:average_revenue, nil) + end + defp sorting_key(metrics) do if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) end @@ -807,8 +775,8 @@ defmodule Plausible.Stats.Breakdown do offset = (page - 1) * limit q - |> Ecto.Query.limit(^limit) - |> Ecto.Query.offset(^offset) + |> limit(^limit) + |> offset(^offset) end defp trace(query, property, metrics) do diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 4c31aabd0..053625a90 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -53,9 +53,8 @@ defmodule Plausible.Stats.Timeseries do defp events_timeseries(_, _, []), do: [] defp events_timeseries(site, query, metrics) do - from(e in base_event_query(site, query), select: %{}) + from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) |> select_bucket(site, query) - |> select_event_metrics(metrics) |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end @@ -63,10 +62,9 @@ defmodule Plausible.Stats.Timeseries do defp sessions_timeseries(_, _, []), do: [] defp sessions_timeseries(site, query, metrics) do - from(e in query_sessions(site, query), select: %{}) + from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) |> filter_converted_sessions(site, query) |> select_bucket(site, query) - |> select_session_metrics(metrics, query) |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() |> Util.keep_requested_metrics(metrics) diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 10078ff6a..85c36888f 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -7,7 +7,8 @@ defmodule Plausible.Stats.Util do :__internal_visits, :visitors, :__total_visitors, - :__breakdown_value + :__breakdown_value, + :total_visitors ] @doc """ @@ -48,12 +49,4 @@ defmodule Plausible.Stats.Util do metrics end end - - def calculate_cr(nil, _converted_visitors), do: nil - - def calculate_cr(unique_visitors, converted_visitors) do - if unique_visitors > 0, - do: Float.round(converted_visitors / unique_visitors * 100, 1), - else: 0.0 - end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 325987def..6f0e35092 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -1569,5 +1569,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do } = json_response(conn, 200)["results"] end + + test "conversion_rate for the filtered goal is 0 when no stats exist", %{ + conn: conn, + site: site + } do + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + get(conn, "/api/v1/stats/aggregate", %{ + "site_id" => site.domain, + "metrics" => "conversion_rate", + "filters" => "event:goal==Signup" + }) + + assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}} + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index a454f59f6..9dd09736e 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -580,6 +580,42 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do } ] end + + test "conversion_rate for goals should not be calculated with imported data", %{ + conn: conn, + site: site + } do + site = + site + |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok") + |> Plausible.Repo.update!() + + populate_stats(site, [ + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/another"), + build(:pageview, pathname: "/blog/post-1"), + build(:pageview, pathname: "/blog/post-2"), + build(:imported_pages, page: "/blog/post-1"), + build(:imported_visitors) + ]) + + insert(:goal, %{site: site, page_path: "/blog**"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/conversions?period=day" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Visit /blog**", + "visitors" => 2, + "events" => 2, + "conversion_rate" => 50 + } + ] + end end describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do