You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
464 lines
12 KiB
Elixir
464 lines
12 KiB
Elixir
defmodule Plausible.Stats.Breakdown do
|
|
use Plausible.ClickhouseRepo
|
|
import Plausible.Stats.Base
|
|
alias Plausible.Stats.Query
|
|
@no_ref "Direct / None"
|
|
|
|
@event_metrics ["visitors", "pageviews", "events"]
|
|
@session_metrics ["visits", "bounce_rate", "visit_duration"]
|
|
@event_props ["event:page", "event:page_match", "event:name"]
|
|
|
|
def breakdown(site, query, "event:goal", metrics, pagination) do
|
|
{event_goals, pageview_goals} =
|
|
Plausible.Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain)
|
|
|> Enum.split_with(fn goal -> goal.event_name end)
|
|
|
|
events = Enum.map(event_goals, & &1.event_name)
|
|
event_query = %Query{query | filters: Map.put(query.filters, "event:name", {:member, events})}
|
|
|
|
event_results =
|
|
if Enum.any?(event_goals) do
|
|
breakdown(site, event_query, "event:name", metrics, pagination)
|
|
|> transform_keys(%{"name" => "goal"})
|
|
else
|
|
[]
|
|
end
|
|
|
|
{limit, page} = pagination
|
|
offset = (page - 1) * limit
|
|
|
|
page_results =
|
|
if Enum.any?(pageview_goals) do
|
|
page_exprs = Enum.map(pageview_goals, & &1.page_path)
|
|
page_regexes = Enum.map(page_exprs, &page_regex/1)
|
|
|
|
from(e in base_event_query(site, query),
|
|
order_by: [desc: fragment("uniq(?)", e.user_id)],
|
|
limit: ^limit,
|
|
offset: ^offset,
|
|
where:
|
|
fragment(
|
|
"notEmpty(multiMatchAllIndices(?, array(?)) as indices)",
|
|
e.pathname,
|
|
^page_regexes
|
|
),
|
|
group_by: fragment("index"),
|
|
select: %{
|
|
"index" => fragment("arrayJoin(indices) as index"),
|
|
"goal" => fragment("concat('Visit ', array(?)[index])", ^page_exprs)
|
|
}
|
|
)
|
|
|> select_event_metrics(metrics)
|
|
|> ClickhouseRepo.all()
|
|
|> Enum.map(fn row -> Map.delete(row, "index") end)
|
|
else
|
|
[]
|
|
end
|
|
|
|
zip_results(event_results, page_results, "event:goal", metrics)
|
|
end
|
|
|
|
def breakdown(site, query, "event:props:" <> custom_prop, metrics, pagination) do
|
|
{limit, _} = pagination
|
|
|
|
none_result =
|
|
if !Enum.any?(query.filters, fn {key, _} -> String.starts_with?(key, "event:props") end) do
|
|
none_filters = Map.put(query.filters, "event:props:" <> custom_prop, {:is, "(none)"})
|
|
none_query = %Query{query | filters: none_filters}
|
|
|
|
from(e in base_event_query(site, none_query),
|
|
order_by: [desc: fragment("uniq(?)", e.user_id)],
|
|
select: %{},
|
|
select_merge: %{^custom_prop => "(none)"},
|
|
having: fragment("uniq(?)", e.user_id) > 0
|
|
)
|
|
|> select_event_metrics(metrics)
|
|
|> ClickhouseRepo.all()
|
|
else
|
|
[]
|
|
end
|
|
|
|
results = breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination)
|
|
zipped = zip_results(none_result, results, custom_prop, metrics)
|
|
|
|
if Enum.find_index(zipped, fn value -> value[custom_prop] == "(none)" end) == limit do
|
|
Enum.slice(zipped, 0..(limit - 1))
|
|
else
|
|
zipped
|
|
end
|
|
end
|
|
|
|
def breakdown(site, query, "event:page", metrics, pagination) do
|
|
event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
|
|
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
|
|
|
|
event_result = breakdown_events(site, query, "event:page", event_metrics, pagination)
|
|
|
|
event_result =
|
|
if "time_on_page" in metrics do
|
|
pages = Enum.map(event_result, & &1["page"])
|
|
time_on_page_result = breakdown_time_on_page(site, query, pages)
|
|
|
|
Enum.map(event_result, fn row ->
|
|
Map.put(row, "time_on_page", time_on_page_result[row["page"]])
|
|
end)
|
|
else
|
|
event_result
|
|
end
|
|
|
|
new_query =
|
|
case event_result do
|
|
[] ->
|
|
query
|
|
|
|
pages ->
|
|
Query.put_filter(query, "visit:entry_page", {:member, Enum.map(pages, & &1["page"])})
|
|
end
|
|
|
|
{limit, _page} = pagination
|
|
|
|
session_result =
|
|
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})
|
|
|> transform_keys(%{"entry_page" => "page"})
|
|
|
|
zip_results(
|
|
event_result,
|
|
session_result,
|
|
"event:page",
|
|
metrics
|
|
)
|
|
end
|
|
|
|
def breakdown(site, query, property, metrics, pagination) when property in @event_props do
|
|
breakdown_events(site, query, property, metrics, pagination)
|
|
end
|
|
|
|
def breakdown(site, query, property, metrics, pagination)
|
|
when property in [
|
|
"visit:source",
|
|
"visit:utm_medium",
|
|
"visit:utm_source",
|
|
"visit:utm_campaign"
|
|
] do
|
|
query = Query.treat_page_filter_as_entry_page(query)
|
|
breakdown_sessions(site, query, property, metrics, pagination)
|
|
end
|
|
|
|
def breakdown(site, query, property, metrics, pagination) do
|
|
breakdown_sessions(site, query, property, metrics, pagination)
|
|
end
|
|
|
|
defp zip_results(event_result, session_result, property, metrics) do
|
|
sort_by = if Enum.member?(metrics, "visitors"), do: "visitors", else: List.first(metrics)
|
|
|
|
property =
|
|
property
|
|
|> String.trim_leading("event:")
|
|
|> String.trim_leading("visit:")
|
|
|> String.trim_leading("props:")
|
|
|
|
null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})
|
|
|
|
prop_values =
|
|
Enum.map(event_result ++ session_result, fn row -> row[property] end)
|
|
|> Enum.uniq()
|
|
|
|
Enum.map(prop_values, fn value ->
|
|
event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}
|
|
session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}
|
|
|
|
Map.merge(null_row, event_row)
|
|
|> Map.merge(session_row)
|
|
end)
|
|
|> Enum.sort_by(fn row -> row[sort_by] end, :desc)
|
|
end
|
|
|
|
defp breakdown_sessions(_, _, _, [], _), do: []
|
|
|
|
defp breakdown_sessions(site, query, property, metrics, {limit, page}) do
|
|
offset = (page - 1) * limit
|
|
|
|
from(s in query_sessions(site, query),
|
|
order_by: [desc: fragment("uniq(?)", s.user_id), asc: fragment("min(?)", s.start)],
|
|
limit: ^limit,
|
|
offset: ^offset,
|
|
select: %{}
|
|
)
|
|
|> filter_converted_sessions(site, query)
|
|
|> do_group_by(property)
|
|
|> select_session_metrics(metrics)
|
|
|> ClickhouseRepo.all()
|
|
end
|
|
|
|
defp breakdown_events(_, _, _, [], _), do: []
|
|
|
|
defp breakdown_events(site, query, property, metrics, {limit, page}) do
|
|
offset = (page - 1) * limit
|
|
|
|
from(e in base_event_query(site, query),
|
|
order_by: [desc: fragment("uniq(?)", e.user_id)],
|
|
limit: ^limit,
|
|
offset: ^offset,
|
|
select: %{}
|
|
)
|
|
|> do_group_by(property)
|
|
|> select_event_metrics(metrics)
|
|
|> ClickhouseRepo.all()
|
|
end
|
|
|
|
defp breakdown_time_on_page(_site, _query, []) do
|
|
[]
|
|
end
|
|
|
|
defp breakdown_time_on_page(site, query, pages) do
|
|
q =
|
|
from(
|
|
e in base_event_query(site, %Query{
|
|
query
|
|
| filters: Map.delete(query.filters, "event:page")
|
|
}),
|
|
select: {
|
|
fragment("? as p", e.pathname),
|
|
fragment("? as t", e.timestamp),
|
|
fragment("? as s", e.session_id)
|
|
},
|
|
order_by: [e.session_id, e.timestamp]
|
|
)
|
|
|
|
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
|
|
|
|
time_query = "
|
|
SELECT
|
|
p,
|
|
round(sum(td)/count(case when p2 != p then 1 end)) as avgTime
|
|
FROM
|
|
(SELECT
|
|
p,
|
|
p2,
|
|
sum(t2-t) as td
|
|
FROM
|
|
(SELECT
|
|
*,
|
|
neighbor(t, 1) as t2,
|
|
neighbor(p, 1) as p2,
|
|
neighbor(s, 1) as s2
|
|
FROM (#{base_query_raw}))
|
|
WHERE s=s2 AND p IN tuple(?)
|
|
GROUP BY p,p2,s)
|
|
GROUP BY p"
|
|
|
|
{:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages])
|
|
res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{})
|
|
end
|
|
|
|
defp do_group_by(
|
|
%Ecto.Query{
|
|
from: %Ecto.Query.FromExpr{source: {"events", _}},
|
|
joins: [%Ecto.Query.JoinExpr{source: {"meta", _}}]
|
|
} = q,
|
|
"event:props:" <> prop
|
|
) do
|
|
from(
|
|
[e, meta] in q,
|
|
group_by: e.name,
|
|
where: meta.key == ^prop,
|
|
group_by: meta.value,
|
|
select_merge: %{^prop => meta.value}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(
|
|
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
|
|
"event:props:" <> prop
|
|
) do
|
|
from(
|
|
e in q,
|
|
inner_lateral_join: meta in fragment("meta"),
|
|
where: meta.key == ^prop,
|
|
group_by: meta.value,
|
|
select_merge: %{^prop => meta.value}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(
|
|
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
|
|
"event:name"
|
|
) do
|
|
from(
|
|
e in q,
|
|
group_by: e.name,
|
|
select_merge: %{"name" => e.name}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(
|
|
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
|
|
"event:page"
|
|
) do
|
|
from(
|
|
e in q,
|
|
group_by: e.pathname,
|
|
select_merge: %{"page" => e.pathname}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(
|
|
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events", _}}} = q,
|
|
"event:page_match"
|
|
) do
|
|
case Map.get(q, :__private_match_sources__) do
|
|
match_exprs when is_list(match_exprs) ->
|
|
from(
|
|
e in q,
|
|
group_by: fragment("index"),
|
|
select_merge: %{
|
|
"index" => fragment("arrayJoin(indices) as index"),
|
|
"page_match" => fragment("array(?)[index]", ^match_exprs)
|
|
}
|
|
)
|
|
end
|
|
end
|
|
|
|
defp do_group_by(q, "visit:source") do
|
|
from(
|
|
s in q,
|
|
group_by: s.referrer_source,
|
|
select_merge: %{
|
|
"source" => fragment("if(empty(?), ?, ?)", s.referrer_source, @no_ref, s.referrer_source)
|
|
}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:country") do
|
|
from(
|
|
s in q,
|
|
group_by: s.country_code,
|
|
where: s.country_code != "\0\0",
|
|
select_merge: %{"country" => s.country_code}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:region") do
|
|
from(
|
|
s in q,
|
|
group_by: s.subdivision1_code,
|
|
where: s.subdivision1_code != "",
|
|
select_merge: %{"region" => s.subdivision1_code}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:city") do
|
|
from(
|
|
s in q,
|
|
group_by: s.city_geoname_id,
|
|
where: s.city_geoname_id != 0,
|
|
select_merge: %{"city" => s.city_geoname_id}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:entry_page") do
|
|
from(
|
|
s in q,
|
|
group_by: s.entry_page,
|
|
select_merge: %{"entry_page" => s.entry_page}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:exit_page") do
|
|
from(
|
|
s in q,
|
|
group_by: s.exit_page,
|
|
select_merge: %{"exit_page" => s.exit_page}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:referrer") do
|
|
from(
|
|
s in q,
|
|
group_by: s.referrer,
|
|
select_merge: %{
|
|
"referrer" => fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)
|
|
}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:utm_medium") do
|
|
from(
|
|
s in q,
|
|
group_by: s.utm_medium,
|
|
select_merge: %{
|
|
"utm_medium" => fragment("if(empty(?), ?, ?)", s.utm_medium, @no_ref, s.utm_medium)
|
|
}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:utm_source") do
|
|
from(
|
|
s in q,
|
|
group_by: s.utm_source,
|
|
select_merge: %{
|
|
"utm_source" => fragment("if(empty(?), ?, ?)", s.utm_source, @no_ref, s.utm_source)
|
|
}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:utm_campaign") do
|
|
from(
|
|
s in q,
|
|
group_by: s.utm_campaign,
|
|
select_merge: %{
|
|
"utm_campaign" => fragment("if(empty(?), ?, ?)", s.utm_campaign, @no_ref, s.utm_campaign)
|
|
}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:device") do
|
|
from(
|
|
s in q,
|
|
group_by: s.screen_size,
|
|
select_merge: %{"device" => s.screen_size}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:os") do
|
|
from(
|
|
s in q,
|
|
group_by: s.operating_system,
|
|
select_merge: %{"os" => s.operating_system}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:os_version") do
|
|
from(
|
|
s in q,
|
|
group_by: s.operating_system_version,
|
|
select_merge: %{"os_version" => s.operating_system_version}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:browser") do
|
|
from(
|
|
s in q,
|
|
group_by: s.browser,
|
|
select_merge: %{"browser" => s.browser}
|
|
)
|
|
end
|
|
|
|
defp do_group_by(q, "visit:browser_version") do
|
|
from(
|
|
s in q,
|
|
group_by: s.browser_version,
|
|
select_merge: %{"browser_version" => s.browser_version}
|
|
)
|
|
end
|
|
|
|
defp transform_keys(results, keys_to_replace) do
|
|
Enum.map(results, fn map ->
|
|
Enum.map(map, fn {key, val} ->
|
|
{Map.get(keys_to_replace, key, key), val}
|
|
end)
|
|
|> Enum.into(%{})
|
|
end)
|
|
end
|
|
end
|