From 253fb5d67d300654cd3afc6757f3783c6a3f6514 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 21 Mar 2024 09:53:31 +0200 Subject: [PATCH] Revert "Move conversion_rate logic from elixir to clickhouse (#3887)" (#3923) This reverts commit 1909743b908980fc22bce4dc17dde88f7529febc. --- extra/lib/plausible/stats/goal/revenue.ex | 20 +- lib/plausible/stats/aggregate.ex | 36 +- lib/plausible/stats/base.ex | 251 +++++++------- lib/plausible/stats/breakdown.ex | 314 ++++++++++-------- lib/plausible/stats/timeseries.ex | 6 +- lib/plausible/stats/util.ex | 11 +- .../aggregate_test.exs | 16 - .../api/stats_controller/conversions_test.exs | 36 -- 8 files changed, 344 insertions(+), 346 deletions(-) diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index 7b6e3368f..46c4049b9 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do @revenue_metrics end - def total_revenue_query() do - dynamic( - [e], - fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + def total_revenue_query(query) do + from(e in query, + select_merge: %{ + total_revenue: + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + } ) end - def average_revenue_query() do - dynamic( - [e], - fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + def average_revenue_query(query) do + from(e in query, + select_merge: %{ + average_revenue: + fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + } ) end diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 68b96183a..56ded33f8 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -11,9 +11,7 @@ defmodule Plausible.Stats.Aggregate do :visitors, :pageviews, :events, - :sample_percent, - :conversion_rate, - :total_visitors + :sample_percent ] ++ @revenue_metrics @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent] @@ -47,6 +45,7 @@ defmodule Plausible.Stats.Aggregate do Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task]) |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end) + |> maybe_put_cr(site, query, metrics) |> Util.keep_requested_metrics(metrics) |> cast_revenue_metrics_to_money(currency) |> Enum.map(&maybe_round_value/1) @@ -54,20 +53,45 @@ defmodule Plausible.Stats.Aggregate do |> Enum.into(%{}) end + defp maybe_put_cr(aggregate_result, site, query, metrics) do + if :conversion_rate in metrics do + all = + query + |> Query.remove_event_filters([:goal, :props]) + |> then(fn query -> aggregate_events(site, query, [:visitors]) end) + |> Map.fetch!(:visitors) + + converted = aggregate_result.visitors + + cr = Util.calculate_cr(all, converted) + + aggregate_result = Map.put(aggregate_result, :conversion_rate, cr) + + if :total_visitors in metrics do + Map.put(aggregate_result, :total_visitors, all) + else + aggregate_result + end + else + aggregate_result + end + end + defp aggregate_events(_, _, []), do: %{} defp aggregate_events(site, query, metrics) do - from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) + from(e in base_event_query(site, query), select: %{}) + |> select_event_metrics(metrics) |> merge_imported(site, query, :aggregate, metrics) - |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported) |> ClickhouseRepo.one() end defp aggregate_sessions(_, _, []), do: %{} defp aggregate_sessions(site, query, metrics) do - from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) + from(e in query_sessions(site, query), select: %{}) |> filter_converted_sessions(site, query) + |> select_session_metrics(metrics, query) |> merge_imported(site, query, :aggregate, metrics) |> ClickhouseRepo.one() |> Util.keep_requested_metrics(metrics) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 463461bf8..83b4ad6d2 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -209,152 +209,163 @@ defmodule Plausible.Stats.Base do def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q - def select_event_metrics(metrics) do - metrics - |> Enum.map(&select_event_metric/1) - |> Enum.reduce(%{}, &Map.merge/2) - end + def select_event_metrics(q, []), do: q - defp select_event_metric(:pageviews) do - %{ - pageviews: - dynamic( - [e], + def select_event_metrics(q, [:pageviews | rest]) do + from(e in q, + select_merge: %{ + pageviews: fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) - ) - } + } + ) + |> select_event_metrics(rest) end - defp select_event_metric(:events) do - %{ - events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))")) - } + def select_event_metrics(q, [:events | rest]) do + from(e in q, + select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")} + ) + |> select_event_metrics(rest) end - defp select_event_metric(:visitors) do - %{ - visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)) - } + def select_event_metrics(q, [:visitors | rest]) do + from(e in q, + select_merge: %{ + visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors) + } + ) + |> select_event_metrics(rest) end on_full_build do - defp select_event_metric(:total_revenue) do - %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()} + def select_event_metrics(q, [:total_revenue | rest]) do + q + |> Plausible.Stats.Goal.Revenue.total_revenue_query() + |> select_event_metrics(rest) end - defp select_event_metric(:average_revenue) do - %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()} + def select_event_metrics(q, [:average_revenue | rest]) do + q + |> Plausible.Stats.Goal.Revenue.average_revenue_query() + |> select_event_metrics(rest) end end - defp select_event_metric(:sample_percent) do - %{ - sample_percent: - dynamic( - [], + def select_event_metrics(q, [:sample_percent | rest]) do + from(e in q, + select_merge: %{ + sample_percent: fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - ) - } + } + ) + |> select_event_metrics(rest) end - defp select_event_metric(:percentage), do: %{} - defp select_event_metric(:conversion_rate), do: %{} - defp select_event_metric(:total_visitors), do: %{} + def select_event_metrics(q, [:percentage | rest]) do + q |> select_event_metrics(rest) + end - defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") + def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}") - def select_session_metrics(metrics, query) do - metrics - |> Enum.map(&select_session_metric(&1, query)) - |> Enum.reduce(%{}, &Map.merge/2) - end + def select_session_metrics(q, [], _query), do: q - defp select_session_metric(:bounce_rate, query) do + def select_session_metrics(q, [:bounce_rate | rest], query) do condition = dynamic_filter_condition(query, "event:page", :entry_page) - %{ - bounce_rate: - dynamic( - [], - fragment( - "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", - ^condition, - ^condition - ) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } + from(s in q, + select_merge: + ^%{ + bounce_rate: + dynamic( + [], + fragment( + "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", + ^condition, + ^condition + ) + ), + __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:visits, _query) do - %{ - visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)) - } + def select_session_metrics(q, [:visits | rest], query) do + from(s in q, + select_merge: %{ + visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign) + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:pageviews, _query) do - %{ - pageviews: - dynamic( - [s], + def select_session_metrics(q, [:pageviews | rest], query) do + from(s in q, + select_merge: %{ + pageviews: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) - ) - } + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:events, _query) do - %{ - events: - dynamic( - [s], - fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) - ) - } + def select_session_metrics(q, [:events | rest], query) do + from(s in q, + select_merge: %{ + events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:visitors, _query) do - %{ - visitors: - dynamic( - [s], + def select_session_metrics(q, [:visitors | rest], query) do + from(s in q, + select_merge: %{ + visitors: selected_as( fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id), :visitors ) - ) - } + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:visit_duration, _query) do - %{ - visit_duration: - dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } + def select_session_metrics(q, [:visit_duration | rest], query) do + from(s in q, + select_merge: %{ + :visit_duration => + fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"), + __internal_visits: fragment("toUInt32(sum(sign))") + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:views_per_visit, _query) do - %{ - views_per_visit: - dynamic( - [s], - fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign) - ), - __internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) - } + def select_session_metrics(q, [:views_per_visit | rest], query) do + from(s in q, + select_merge: %{ + views_per_visit: + fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign), + __internal_visits: fragment("toUInt32(sum(sign))") + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:sample_percent, _query) do - %{ - sample_percent: - dynamic( - [], + def select_session_metrics(q, [:sample_percent | rest], query) do + from(e in q, + select_merge: %{ + sample_percent: fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") - ) - } + } + ) + |> select_session_metrics(rest, query) end - defp select_session_metric(:percentage, _query), do: %{} + def select_session_metrics(q, [:percentage | rest], query) do + q |> select_session_metrics(rest, query) + end def dynamic_filter_condition(query, filter_key, db_field) do case query && query.filters && query.filters[filter_key] do @@ -575,7 +586,7 @@ defmodule Plausible.Stats.Base do |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) end - defp total_visitors_subquery(site, query, true) do + defp total_visitors_subquery(site, %Query{include_imported: true} = query) do dynamic( [e], selected_as( @@ -586,16 +597,14 @@ defmodule Plausible.Stats.Base do ) end - defp total_visitors_subquery(site, query, false) do + defp total_visitors_subquery(site, query) do dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) end def add_percentage_metric(q, site, query, metrics) do if :percentage in metrics do q - |> select_merge( - ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)} - ) + |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)}) |> select_merge(%{ percentage: fragment( @@ -609,32 +618,4 @@ defmodule Plausible.Stats.Base do q end end - - # Adds conversion_rate metric to query, calculated as - # X / Y where Y is the same breakdown value without goal or props - # filters. - def maybe_add_conversion_rate(q, site, query, metrics, opts) do - if :conversion_rate in metrics do - include_imported = Keyword.fetch!(opts, :include_imported) - - total_query = query |> Query.remove_event_filters([:goal, :props]) - - # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL - subquery(q) - |> select_merge( - ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)} - ) - |> select_merge([e], %{ - conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - selected_as(:__total_visitors), - e.visitors, - selected_as(:__total_visitors) - ) - }) - else - q - end - end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 822c00df1..ae33e4e73 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -4,7 +4,6 @@ defmodule Plausible.Stats.Breakdown do use Plausible.Stats.Fragments import Plausible.Stats.{Base, Imported} - import Ecto.Query require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Query, Util} @@ -53,74 +52,50 @@ defmodule Plausible.Stats.Breakdown do metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - event_q = + event_results = if Enum.any?(event_goals) do site - |> breakdown_events(event_query, "event:name", metrics_to_select) - |> apply_pagination(pagination) + |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true) + |> transform_keys(%{name: :goal}) + |> cast_revenue_metrics_to_money(revenue_goals) else - nil + [] end - page_q = + {limit, page} = pagination + offset = (page - 1) * limit + + page_results = if Enum.any?(pageview_goals) do page_exprs = Enum.map(pageview_goals, & &1.page_path) page_regexes = Enum.map(page_exprs, &page_regex/1) - select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil - from(e in base_event_query(site, query), order_by: [desc: fragment("uniq(?)", e.user_id)], + limit: ^limit, + offset: ^offset, where: fragment( "notEmpty(multiMatchAllIndices(?, ?) as indices)", e.pathname, ^page_regexes ) and e.name == "pageview", - array_join: index in fragment("indices"), - group_by: index, - select: %{} + group_by: fragment("index"), + select: %{ + index: fragment("arrayJoin(indices) as index"), + goal: fragment("concat('Visit ', ?[index])", ^page_exprs) + } ) - |> select_merge(^select_columns) - # :TRICKY: name is added last to make sure both queries add columns in the same order - |> select_merge([_, index], %{ - name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) - }) - |> apply_pagination(pagination) + |> select_event_metrics(metrics_to_select -- @revenue_metrics) + |> ClickhouseRepo.all() + |> Enum.map(fn row -> Map.delete(row, :index) end) else - nil - end - - full_q = - case {event_q, page_q} do - {nil, nil} -> - nil - - {event_q, nil} -> - event_q - - {nil, page_q} -> - page_q - - {event_q, page_q} -> - from( - e in subquery(union_all(event_q, ^page_q)), - # :TODO: Handle other orderings - order_by: [desc: e.visitors] - ) - |> apply_pagination(pagination) + [] end - if full_q do - full_q - |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) - |> ClickhouseRepo.all() - |> transform_keys(%{name: :goal}) - |> cast_revenue_metrics_to_money(revenue_goals) - |> Util.keep_requested_metrics(metrics) - else - [] - end + zip_results(event_results, page_results, :goal, metrics_to_select) + |> maybe_add_cr(site, query, nil, metrics) + |> Util.keep_requested_metrics(metrics) end def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do @@ -135,11 +110,11 @@ defmodule Plausible.Stats.Breakdown do if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) - breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select) - |> maybe_add_conversion_rate(site, query, metrics, include_imported: false) - |> paginate_and_execute(metrics, pagination) - |> transform_keys(%{name: custom_prop}) + breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination) |> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) + |> sort_results(metrics_to_select) + |> maybe_add_cr(site, query, nil, metrics) + |> Util.keep_requested_metrics(metrics) end def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do @@ -150,10 +125,10 @@ defmodule Plausible.Stats.Breakdown do event_result = site - |> breakdown_events(query, property, event_metrics) - |> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics) - |> paginate_and_execute(metrics, pagination) + |> breakdown_events(query, "event:page", event_metrics, pagination) |> maybe_add_time_on_page(site, query, metrics) + |> maybe_add_cr(site, query, property, metrics, pagination) + |> Util.keep_requested_metrics(metrics) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) @@ -174,8 +149,7 @@ defmodule Plausible.Stats.Breakdown do {limit, _page} = pagination session_result = - breakdown_sessions(site, new_query, "visit:entry_page", session_metrics) - |> paginate_and_execute(session_metrics, {limit, 1}) + breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1}) |> transform_keys(%{entry_page: :page}) metrics = metrics ++ [:page] @@ -192,9 +166,7 @@ defmodule Plausible.Stats.Breakdown do def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) - - breakdown_events(site, query, property, metrics) - |> paginate_and_execute(metrics, pagination) + breakdown_events(site, query, property, metrics, pagination) end def breakdown(site, query, property, metrics, pagination, opts) do @@ -202,9 +174,9 @@ defmodule Plausible.Stats.Breakdown do metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics - breakdown_sessions(site, query, property, metrics_to_select) - |> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics) - |> paginate_and_execute(metrics, pagination) + breakdown_sessions(site, query, property, metrics_to_select, pagination) + |> maybe_add_cr(site, query, property, metrics, pagination) + |> Util.keep_requested_metrics(metrics) end defp zip_results(event_result, session_result, property, metrics) do @@ -225,31 +197,35 @@ defmodule Plausible.Stats.Breakdown do |> sort_results(metrics) end - defp breakdown_sessions(site, query, property, metrics) do + defp breakdown_sessions(_, _, _, [], _), do: [] + + defp breakdown_sessions(site, query, property, metrics, pagination) do from(s in query_sessions(site, query), order_by: [desc: fragment("uniq(?)", s.user_id)], - select: ^select_session_metrics(metrics, query) + select: %{} ) |> filter_converted_sessions(site, query) |> do_group_by(property) + |> select_session_metrics(metrics, query) |> merge_imported(site, query, property, metrics) |> add_percentage_metric(site, query, metrics) + |> apply_pagination(pagination) + |> ClickhouseRepo.all() + |> transform_keys(%{operating_system: :os}) + |> Util.keep_requested_metrics(metrics) end - defp breakdown_events(site, query, property, metrics) do + defp breakdown_events(_, _, _, [], _), do: [] + + defp breakdown_events(site, query, property, metrics, pagination) do from(e in base_event_query(site, query), order_by: [desc: fragment("uniq(?)", e.user_id)], - select: ^select_event_metrics(metrics) + select: %{} ) |> do_group_by(property) + |> select_event_metrics(metrics) |> merge_imported(site, query, property, metrics) |> add_percentage_metric(site, query, metrics) - end - - defp paginate_and_execute(_, [], _), do: [] - - defp paginate_and_execute(q, metrics, pagination) do - q |> apply_pagination(pagination) |> ClickhouseRepo.all() |> transform_keys(%{operating_system: :os}) @@ -447,18 +423,18 @@ defmodule Plausible.Stats.Breakdown do from( e in q, select_merge: %{ - name: + ^prop => selected_as( fragment( "if(not empty(?), ?, '(none)')", get_by_key(e, :meta, ^prop), get_by_key(e, :meta, ^prop) ), - :name + :breakdown_prop_value ) }, - group_by: selected_as(:name), - order_by: {:asc, selected_as(:name)} + group_by: selected_as(:breakdown_prop_value), + order_by: {:asc, selected_as(:breakdown_prop_value)} ) end @@ -676,26 +652,130 @@ defmodule Plausible.Stats.Breakdown do ) end - defp group_by_field_names("event:props:" <> _prop), do: [:name] - defp group_by_field_names("visit:os"), do: [:operating_system] - defp group_by_field_names("visit:os_version"), do: [:os, :os_version] - defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version] + defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do + cond do + :conversion_rate not in metrics -> breakdown_results + Enum.empty?(breakdown_results) -> breakdown_results + is_nil(property) -> add_absolute_cr(breakdown_results, site, query) + true -> add_cr(breakdown_results, site, query, property, metrics, pagination) + end + end + + # This function injects a conversion_rate metric into every + # breakdown result map. It is calculated as X / Y, where: + # + # * X is the number of conversions for a breakdown + # result (conversion = number of visitors who + # completed the filtered goal with the filtered + # custom properties). + # + # * Y is the number of all visitors for this breakdown + # result without the `event:goal` and `event:props:*` + # filters. + defp add_cr(breakdown_results, site, query, property, metrics, pagination) do + property_atom = Plausible.Stats.Filters.without_prefix(property) + + items = + Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end) + + query_without_goal = + query + |> Query.put_filter(property, {:member, items}) + |> Query.remove_event_filters([:goal, :props]) + + # Here, we're always only interested in the first page of results + # - the :member filter makes sure that the results always match with + # the items in the given breakdown_results list + page = 1 + + # For browser/os versions we need to fetch a lot more entries than the + # pagination limit. This is because many entries can correspond to a + # single version number and we need to make sure that the results + # without goal filter will include all those combinations of browsers/os-s + # and their versions that were present in the `breakdown_results` array. + {pagination_limit, find_match_fn} = + case property_atom do + :browser_version -> + pagination_limit = min(elem(pagination, 0) * 10, 3_000) + + find_match_fn = fn total, conversion -> + total[:browser_version] == conversion[:browser_version] && + total[:browser] == conversion[:browser] + end + + {pagination_limit, find_match_fn} + + :os_version -> + pagination_limit = min(elem(pagination, 0) * 5, 3_000) + + find_match_fn = fn total, conversion -> + total[:os_version] == conversion[:os_version] && + total[:os] == conversion[:os] + end + + {pagination_limit, find_match_fn} + + _ -> + {elem(pagination, 0), + fn total, conversion -> + total[property_atom] == conversion[property_atom] + end} + end + + pagination = {pagination_limit, page} - defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)] + res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination) - defp on_matches_group_by(fields) do - Enum.reduce(fields, nil, &fields_equal/2) - end + Enum.map(breakdown_results, fn item -> + without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item)) - defp outer_order_by(fields) do - Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end) + {conversion_rate, total_visitors} = + if without_goal do + {Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors} + else + Sentry.capture_message( + "Unable to find a conversion_rate divisor from a breakdown response", + extra: %{ + domain: site.domain, + query: inspect(query), + property: property, + pagination: inspect(pagination), + item_not_found: inspect(item) + } + ) + + {"N/A", "N/A"} + end + + if :total_visitors in metrics do + item + |> Map.put(:conversion_rate, conversion_rate) + |> Map.put(:total_visitors, total_visitors) + else + Map.put(item, :conversion_rate, conversion_rate) + end + end) end - defp fields_equal(field_name, nil), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name)) + # Similar to `add_cr/5`, injects a conversion_rate metric into + # every breakdown result. However, a single divisor is used in + # the CR calculation across all breakdown results. That is the + # number of visitors without `event:goal` and `event:props:*` + # filters. + # + # This is useful when we're only interested in the conversions + # themselves - not how well a certain property such as browser + # or page converted. + defp add_absolute_cr(breakdown_results, site, query) do + total_q = Query.remove_event_filters(query, [:goal, :props]) + + %{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors]) - defp fields_equal(field_name, condition), - do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition) + breakdown_results + |> Enum.map(fn goal -> + Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors])) + end) + end defp sort_results(results, metrics) do Enum.sort_by( @@ -710,54 +790,6 @@ defmodule Plausible.Stats.Breakdown do ) end - # This function injects a conversion_rate metric into - # a breakdown query. It is calculated as X / Y, where: - # - # * X is the number of conversions for a breakdown - # result (conversion = number of visitors who - # completed the filtered goal with the filtered - # custom properties). - # - # * Y is the number of all visitors for this breakdown - # result without the `event:goal` and `event:props:*` - # filters. - defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do - if :conversion_rate in metrics do - breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props]) - - breakdown_total_visitors_q = - breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors]) - - from(e in subquery(q), - left_join: c in subquery(breakdown_total_visitors_q), - on: ^on_matches_group_by(group_by_field_names(property)), - select_merge: %{ - total_visitors: c.visitors, - conversion_rate: - fragment( - "if(? > 0, round(? / ? * 100, 1), 0)", - c.visitors, - e.visitors, - c.visitors - ) - }, - order_by: [desc: e.visitors], - order_by: ^outer_order_by(group_by_field_names(property)) - ) - else - q - end - end - - # When querying custom event goals and pageviewgoals together, UNION ALL is used - # so the same fields must be present on both sides of the union. This change to the - # query will ensure that we don't unnecessarily read revenue column for pageview goals - defp mark_revenue_as_nil(select_columns) do - select_columns - |> Map.replace(:total_revenue, nil) - |> Map.replace(:average_revenue, nil) - end - defp sorting_key(metrics) do if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) end @@ -775,8 +807,8 @@ defmodule Plausible.Stats.Breakdown do offset = (page - 1) * limit q - |> limit(^limit) - |> offset(^offset) + |> Ecto.Query.limit(^limit) + |> Ecto.Query.offset(^offset) end defp trace(query, property, metrics) do diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 053625a90..4c31aabd0 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -53,8 +53,9 @@ defmodule Plausible.Stats.Timeseries do defp events_timeseries(_, _, []), do: [] defp events_timeseries(site, query, metrics) do - from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) + from(e in base_event_query(site, query), select: %{}) |> select_bucket(site, query) + |> select_event_metrics(metrics) |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end @@ -62,9 +63,10 @@ defmodule Plausible.Stats.Timeseries do defp sessions_timeseries(_, _, []), do: [] defp sessions_timeseries(site, query, metrics) do - from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) + from(e in query_sessions(site, query), select: %{}) |> filter_converted_sessions(site, query) |> select_bucket(site, query) + |> select_session_metrics(metrics, query) |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() |> Util.keep_requested_metrics(metrics) diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex index 85c36888f..10078ff6a 100644 --- a/lib/plausible/stats/util.ex +++ b/lib/plausible/stats/util.ex @@ -7,8 +7,7 @@ defmodule Plausible.Stats.Util do :__internal_visits, :visitors, :__total_visitors, - :__breakdown_value, - :total_visitors + :__breakdown_value ] @doc """ @@ -49,4 +48,12 @@ defmodule Plausible.Stats.Util do metrics end end + + def calculate_cr(nil, _converted_visitors), do: nil + + def calculate_cr(unique_visitors, converted_visitors) do + if unique_visitors > 0, + do: Float.round(converted_visitors / unique_visitors * 100, 1), + else: 0.0 + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 6f0e35092..325987def 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -1569,21 +1569,5 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do } = json_response(conn, 200)["results"] end - - test "conversion_rate for the filtered goal is 0 when no stats exist", %{ - conn: conn, - site: site - } do - insert(:goal, %{site: site, event_name: "Signup"}) - - conn = - get(conn, "/api/v1/stats/aggregate", %{ - "site_id" => site.domain, - "metrics" => "conversion_rate", - "filters" => "event:goal==Signup" - }) - - assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}} - end end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 9dd09736e..a454f59f6 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -580,42 +580,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do } ] end - - test "conversion_rate for goals should not be calculated with imported data", %{ - conn: conn, - site: site - } do - site = - site - |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok") - |> Plausible.Repo.update!() - - populate_stats(site, [ - build(:pageview, pathname: "/"), - build(:pageview, pathname: "/another"), - build(:pageview, pathname: "/blog/post-1"), - build(:pageview, pathname: "/blog/post-2"), - build(:imported_pages, page: "/blog/post-1"), - build(:imported_visitors) - ]) - - insert(:goal, %{site: site, page_path: "/blog**"}) - - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "Visit /blog**", - "visitors" => 2, - "events" => 2, - "conversion_rate" => 50 - } - ] - end end describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do