Compare commits

...

17 Commits

Author SHA1 Message Date
Uku Taht 167fb9dabf Refactor attempt 3 years ago
Karl-Aksel Puulmann e4acdfd7d8 Solve rebase fail 3 years ago
Karl-Aksel Puulmann 797e272ec6 Handle conversion_rate in aggregate.ex 3 years ago
Karl-Aksel Puulmann e48a7c236a handle total_visitors the same way as add_percentage 3 years ago
Karl-Aksel Puulmann 6c0389a458 Import Ecto.Query and cleanup 3 years ago
Karl-Aksel Puulmann cb7d8975f5 Compatibility fix 3 years ago
Karl-Aksel Puulmann a6dbf68642 CR with order_by 3 years ago
Karl-Aksel Puulmann bcb749aabe WIP: Calculate group CR 3 years ago
Karl-Aksel Puulmann c23274b4b9 Calculate CR for property breakdowns 3 years ago
Karl-Aksel Puulmann 6489be416b Calculate CR within the goal breakdown query 3 years ago
Karl-Aksel Puulmann 8a19f12c84 Make sure column order is identical 3 years ago
Karl-Aksel Puulmann c78a7cbd7a Order in DB 3 years ago
Karl-Aksel Puulmann 03e54d7602 Do everything in a single query for breakdowns for goals 3 years ago
Karl-Aksel Puulmann b0b3e5e895 Use common pagination util 3 years ago
Karl-Aksel Puulmann c004a5edfa Remove zip_columns logic 3 years ago
Karl-Aksel Puulmann 556cafee8f Refactor pageview_goals breakdown query, removing index column from results 3 years ago
Karl-Aksel Puulmann 55b759624e Separate out query building from pagination/execution logic. 3 years ago

@ -11,7 +11,9 @@ defmodule Plausible.Stats.Aggregate do
:visitors,
:pageviews,
:events,
:sample_percent
:sample_percent,
:conversion_rate,
:total_visitors
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
@ -45,7 +47,6 @@ defmodule Plausible.Stats.Aggregate do
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> maybe_put_cr(site, query, metrics)
|> Util.keep_requested_metrics(metrics)
|> cast_revenue_metrics_to_money(currency)
|> Enum.map(&maybe_round_value/1)
@ -53,36 +54,13 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{})
end
defp maybe_put_cr(aggregate_result, site, query, metrics) do
if :conversion_rate in metrics do
all =
query
|> Query.remove_event_filters([:goal, :props])
|> then(fn query -> aggregate_events(site, query, [:visitors]) end)
|> Map.fetch!(:visitors)
converted = aggregate_result.visitors
cr = Util.calculate_cr(all, converted)
aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)
if :total_visitors in metrics do
Map.put(aggregate_result, :total_visitors, all)
else
aggregate_result
end
else
aggregate_result
end
end
defp aggregate_events(_, _, []), do: %{}
defp aggregate_events(site, query, metrics) do
from(e in base_event_query(site, query), select: %{})
|> select_event_metrics(metrics)
|> merge_imported(site, query, :aggregate, metrics)
|> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.one()
end

@ -265,6 +265,14 @@ defmodule Plausible.Stats.Base do
q |> select_event_metrics(rest)
end
def select_event_metrics(q, [:conversion_rate | rest]) do
q |> select_event_metrics(rest)
end
def select_event_metrics(q, [:total_visitors | rest]) do
q |> select_event_metrics(rest)
end
def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}")
def select_session_metrics(q, [], _query), do: q
@ -586,7 +594,7 @@ defmodule Plausible.Stats.Base do
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
end
defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
def total_visitors_subquery(site, %Query{include_imported: true} = query) do
dynamic(
[e],
selected_as(
@ -597,7 +605,7 @@ defmodule Plausible.Stats.Base do
)
end
defp total_visitors_subquery(site, query) do
def total_visitors_subquery(site, query) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end
@ -618,4 +626,28 @@ defmodule Plausible.Stats.Base do
q
end
end
# Adds conversion_rate metric to query, calculated as
# X / Y where Y is the same breakdown value without goal or props
# filters.
def maybe_add_conversion_rate(q, site, query, metrics) do
if :conversion_rate in metrics do
total_query = query |> Query.remove_event_filters([:goal, :props])
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
|> select_merge(^%{total_visitors: total_visitors_subquery(site, total_query)})
|> select_merge([e], %{
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
)
})
else
q
end
end
end

@ -4,6 +4,7 @@ defmodule Plausible.Stats.Breakdown do
use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported}
import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util}
@ -52,50 +53,71 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
event_results =
event_q =
if Enum.any?(event_goals) do
site
|> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> breakdown_events(event_query, "event:name", metrics_to_select)
|> apply_pagination(pagination)
else
[]
nil
end
{limit, page} = pagination
offset = (page - 1) * limit
page_results =
page_q =
if Enum.any?(pageview_goals) do
page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1)
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
limit: ^limit,
offset: ^offset,
where:
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
) and e.name == "pageview",
group_by: fragment("index"),
array_join: index in fragment("indices"),
group_by: index,
select: %{
index: fragment("arrayJoin(indices) as index"),
goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
}
)
|> select_event_metrics(metrics_to_select -- @revenue_metrics)
|> ClickhouseRepo.all()
|> Enum.map(fn row -> Map.delete(row, :index) end)
|> select_event_metrics(metrics_to_select)
|> revenue_is_nil(metrics_to_select)
|> apply_pagination(pagination)
else
[]
nil
end
zip_results(event_results, page_results, :goal, metrics_to_select)
|> maybe_add_cr(site, query, nil, metrics)
|> Util.keep_requested_metrics(metrics)
full_q =
case {event_q, page_q} do
{nil, nil} ->
nil
{event_q, nil} ->
event_q
{nil, page_q} ->
page_q
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
# :TODO: Handle other orderings
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
end
if full_q do
full_q
|> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
else
[]
end
end
def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do
@ -110,11 +132,11 @@ defmodule Plausible.Stats.Breakdown do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)
breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select)
|> maybe_add_conversion_rate(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{name: custom_prop})
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
|> sort_results(metrics_to_select)
|> maybe_add_cr(site, query, nil, metrics)
|> Util.keep_requested_metrics(metrics)
end
def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do
@ -125,10 +147,10 @@ defmodule Plausible.Stats.Breakdown do
event_result =
site
|> breakdown_events(query, "event:page", event_metrics, pagination)
|> breakdown_events(query, property, event_metrics)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics)
|> maybe_add_cr(site, query, property, metrics, pagination)
|> Util.keep_requested_metrics(metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
@ -149,7 +171,8 @@ defmodule Plausible.Stats.Breakdown do
{limit, _page} = pagination
session_result =
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics)
|> paginate_and_execute(session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page]
@ -166,7 +189,9 @@ defmodule Plausible.Stats.Breakdown do
def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, property, metrics, pagination)
breakdown_events(site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end
def breakdown(site, query, property, metrics, pagination, opts) do
@ -174,9 +199,9 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
breakdown_sessions(site, query, property, metrics_to_select, pagination)
|> maybe_add_cr(site, query, property, metrics, pagination)
|> Util.keep_requested_metrics(metrics)
breakdown_sessions(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end
defp zip_results(event_result, session_result, property, metrics) do
@ -197,9 +222,7 @@ defmodule Plausible.Stats.Breakdown do
|> sort_results(metrics)
end
defp breakdown_sessions(_, _, _, [], _), do: []
defp breakdown_sessions(site, query, property, metrics, pagination) do
defp breakdown_sessions(site, query, property, metrics) do
from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)],
select: %{}
@ -209,15 +232,9 @@ defmodule Plausible.Stats.Breakdown do
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
|> Util.keep_requested_metrics(metrics)
end
defp breakdown_events(_, _, _, [], _), do: []
defp breakdown_events(site, query, property, metrics, pagination) do
defp breakdown_events(site, query, property, metrics) do
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
select: %{}
@ -226,6 +243,12 @@ defmodule Plausible.Stats.Breakdown do
|> select_event_metrics(metrics)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
end
defp paginate_and_execute(_, [], _), do: []
defp paginate_and_execute(q, metrics, pagination) do
q
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
@ -423,18 +446,18 @@ defmodule Plausible.Stats.Breakdown do
from(
e in q,
select_merge: %{
^prop =>
name:
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(e, :meta, ^prop),
get_by_key(e, :meta, ^prop)
),
:breakdown_prop_value
:name
)
},
group_by: selected_as(:breakdown_prop_value),
order_by: {:asc, selected_as(:breakdown_prop_value)}
group_by: selected_as(:name),
order_by: {:asc, selected_as(:name)}
)
end
@ -652,130 +675,26 @@ defmodule Plausible.Stats.Breakdown do
)
end
defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do
cond do
:conversion_rate not in metrics -> breakdown_results
Enum.empty?(breakdown_results) -> breakdown_results
is_nil(property) -> add_absolute_cr(breakdown_results, site, query)
true -> add_cr(breakdown_results, site, query, property, metrics, pagination)
end
end
# This function injects a conversion_rate metric into every
# breakdown result map. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
defp add_cr(breakdown_results, site, query, property, metrics, pagination) do
property_atom = Plausible.Stats.Filters.without_prefix(property)
items =
Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)
query_without_goal =
query
|> Query.put_filter(property, {:member, items})
|> Query.remove_event_filters([:goal, :props])
# Here, we're always only interested in the first page of results
# - the :member filter makes sure that the results always match with
# the items in the given breakdown_results list
page = 1
# For browser/os versions we need to fetch a lot more entries than the
# pagination limit. This is because many entries can correspond to a
# single version number and we need to make sure that the results
# without goal filter will include all those combinations of browsers/os-s
# and their versions that were present in the `breakdown_results` array.
{pagination_limit, find_match_fn} =
case property_atom do
:browser_version ->
pagination_limit = min(elem(pagination, 0) * 10, 3_000)
find_match_fn = fn total, conversion ->
total[:browser_version] == conversion[:browser_version] &&
total[:browser] == conversion[:browser]
end
{pagination_limit, find_match_fn}
:os_version ->
pagination_limit = min(elem(pagination, 0) * 5, 3_000)
find_match_fn = fn total, conversion ->
total[:os_version] == conversion[:os_version] &&
total[:os] == conversion[:os]
end
{pagination_limit, find_match_fn}
_ ->
{elem(pagination, 0),
fn total, conversion ->
total[property_atom] == conversion[property_atom]
end}
end
pagination = {pagination_limit, page}
res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)
Enum.map(breakdown_results, fn item ->
without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))
defp group_by_field_names("event:props:" <> _prop), do: [:name]
defp group_by_field_names("visit:os"), do: [:operating_system]
defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
{conversion_rate, total_visitors} =
if without_goal do
{Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}
else
Sentry.capture_message(
"Unable to find a conversion_rate divisor from a breakdown response",
extra: %{
domain: site.domain,
query: inspect(query),
property: property,
pagination: inspect(pagination),
item_not_found: inspect(item)
}
)
{"N/A", "N/A"}
end
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
if :total_visitors in metrics do
item
|> Map.put(:conversion_rate, conversion_rate)
|> Map.put(:total_visitors, total_visitors)
else
Map.put(item, :conversion_rate, conversion_rate)
end
end)
defp on_matches_group_by(fields) do
Enum.reduce(fields, nil, &fields_equal/2)
end
# Similar to `add_cr/5`, injects a conversion_rate metric into
# every breakdown result. However, a single divisor is used in
# the CR calculation across all breakdown results. That is the
# number of visitors without `event:goal` and `event:props:*`
# filters.
#
# This is useful when we're only interested in the conversions
# themselves - not how well a certain property such as browser
# or page converted.
defp add_absolute_cr(breakdown_results, site, query) do
total_q = Query.remove_event_filters(query, [:goal, :props])
def outer_order_by(fields) do
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
end
%{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])
defp fields_equal(field_name, nil),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
breakdown_results
|> Enum.map(fn goal ->
Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))
end)
end
defp fields_equal(field_name, condition),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
defp sort_results(results, metrics) do
Enum.sort_by(
@ -790,6 +709,48 @@ defmodule Plausible.Stats.Breakdown do
)
end
# This function injects a conversion_rate metric into
# a breakdown query. It is calculated as X / Y, where:
#
# * X is the number of conversions for a breakdown
# result (conversion = number of visitors who
# completed the filtered goal with the filtered
# custom properties).
#
# * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*`
# filters.
def maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do
if :conversion_rate in metrics do
event_metrics =
metrics
|> Util.maybe_add_visitors_metric()
|> Enum.filter(&(&1 in @event_metrics))
groups_totals_query = query |> Query.remove_event_filters([:goal, :props])
groups_breakdown_q = breakdown_fn.(site, groups_totals_query, property, event_metrics)
from(e in subquery(q),
left_join: c in subquery(groups_breakdown_q),
on: ^on_matches_group_by(group_by_field_names(property)),
select_merge: %{
total_visitors: c.visitors,
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), null)",
c.visitors,
e.visitors,
c.visitors
)
},
order_by: [desc: e.visitors],
order_by: ^outer_order_by(group_by_field_names(property))
)
else
q
end
end
defp sorting_key(metrics) do
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
end
@ -807,8 +768,8 @@ defmodule Plausible.Stats.Breakdown do
offset = (page - 1) * limit
q
|> Ecto.Query.limit(^limit)
|> Ecto.Query.offset(^offset)
|> limit(^limit)
|> offset(^offset)
end
defp trace(query, property, metrics) do
@ -819,6 +780,25 @@ defmodule Plausible.Stats.Breakdown do
])
end
# When querying custom event goals and pageviewgoals together, UNION ALL is used
# so the same fields must be present on both sides of the union. This change to the
# query will ensure that we don't unnecessarily read revenue column for pageview goals
defp revenue_is_nil(q, metrics) do
overwrite_fields = %{}
overwrite_fields =
if :total_revenue in metrics,
do: Map.merge(overwrite_fields, %{total_revenue: nil}),
else: overwrite_fields
overwrite_fields =
if :average_revenue in metrics,
do: Map.merge(overwrite_fields, %{average_revenue: nil}),
else: overwrite_fields
from(e in q, select_merge: ^overwrite_fields)
end
on_full_build do
defp cast_revenue_metrics_to_money(results, revenue_goals) do
Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)

@ -7,7 +7,8 @@ defmodule Plausible.Stats.Util do
:__internal_visits,
:visitors,
:__total_visitors,
:__breakdown_value
:__breakdown_value,
:total_visitors
]
@doc """
@ -48,12 +49,4 @@ defmodule Plausible.Stats.Util do
metrics
end
end
def calculate_cr(nil, _converted_visitors), do: nil
def calculate_cr(unique_visitors, converted_visitors) do
if unique_visitors > 0,
do: Float.round(converted_visitors / unique_visitors * 100, 1),
else: 0.0
end
end

Loading…
Cancel
Save