use windows in 'time on page' queries (#3446)

* use windows in 'time on page' query

* add imported time on page back

* join imported_pages instead of extra query

* don't forget aggregated time on page!

* eh?

* no need for separate window clause

* use dynamic_filter_condition in aggregate_time_on_page

* it's avg, not total

* fix dynamic_filter_condition call

* tests pass

* fewer changes

* fewer changes + feature flag

* base quickfix

* fewer changes

* fewer changes?

* fewer changes!

* replace coalesce with if(empty(),etc)

* make window_aggregate_time_on_page return same result as neighbor_

* breakdown as well

* add failing test

* more info

* more info

* format example

* add failing aggregate.ex test

* fix breakdown.ex windowing time_on_page calculation

* fix aggregate.ex windowing time_on_page calculation
pull/3526/head
ruslandoga 3 years ago committed by GitHub
parent dc5ab2e407
commit 13055aafc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -58,6 +58,14 @@ defmodule Plausible.Stats.Aggregate do
end
defp aggregate_time_on_page(site, query) do
if FunWithFlags.enabled?(:window_time_on_page) do
window_aggregate_time_on_page(site, query)
else
neighbor_aggregate_time_on_page(site, query)
end
end
defp neighbor_aggregate_time_on_page(site, query) do
q =
from(
e in base_event_query(site, %Query{
@ -136,6 +144,50 @@ defmodule Plausible.Stats.Aggregate do
%{time_on_page: time_on_page}
end
defp window_aggregate_time_on_page(site, query) do
windowed_pages_q =
from e in base_event_query(site, %Query{
query
| filters: Map.delete(query.filters, "event:page")
}),
select: %{
next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
timestamp: e.timestamp,
pathname: e.pathname,
session_id: e.session_id
},
windows: [
event_horizon: [
partition_by: e.session_id,
order_by: e.timestamp,
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
]
]
timed_page_transitions_q =
from e in Ecto.Query.subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,
transition: e.next_pathname != e.pathname,
duration: sum(e.next_timestamp - e.timestamp)
}
avg_time_per_page_transition_q =
from e in Ecto.Query.subquery(timed_page_transitions_q),
select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)},
group_by: e.pathname
time_on_page_q =
from e in Ecto.Query.subquery(avg_time_per_page_transition_q),
select: fragment("avg(ifNotFinite(?,NULL))", e.avg)
%{time_on_page: ClickhouseRepo.one(time_on_page_q)}
end
@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
defp maybe_round_value({metric, nil}), do: {metric, 0}

@ -425,7 +425,7 @@ defmodule Plausible.Stats.Base do
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
defp dynamic_filter_condition(query, filter_key, db_field) do
def dynamic_filter_condition(query, filter_key, db_field) do
case query && query.filters && query.filters[filter_key] do
{:is, value} ->
value = db_field_val(db_field, value)

@ -211,10 +211,18 @@ defmodule Plausible.Stats.Breakdown do
end
defp breakdown_time_on_page(_site, _query, []) do
[]
%{}
end
defp breakdown_time_on_page(site, query, pages) do
if FunWithFlags.enabled?(:window_time_on_page) do
window_breakdown_time_on_page(site, query, pages)
else
neighbor_breakdown_time_on_page(site, query, pages)
end
end
defp neighbor_breakdown_time_on_page(site, query, pages) do
q =
from(
e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
@ -294,6 +302,85 @@ defmodule Plausible.Stats.Breakdown do
end
end
defp window_breakdown_time_on_page(site, query, pages) do
import Ecto.Query
windowed_pages_q =
from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
select: %{
next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
timestamp: e.timestamp,
pathname: e.pathname,
session_id: e.session_id
},
windows: [
event_horizon: [
partition_by: e.session_id,
order_by: e.timestamp,
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
]
]
timed_page_transitions_q =
from e in subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
where: e.pathname in ^pages,
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,
transition: e.next_pathname != e.pathname,
duration: sum(e.next_timestamp - e.timestamp)
}
no_select_timed_pages_q =
from e in subquery(timed_page_transitions_q),
group_by: e.pathname
timed_pages_q =
if query.include_imported do
# Imported page views have pre-calculated values
imported_timed_pages_q =
from i in "imported_pages",
group_by: i.page,
where: i.site_id == ^site.id,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.page in ^pages,
select: %{
page: i.page,
time_on_page: sum(i.time_on_page),
visits: sum(i.pageviews) - sum(i.exits)
}
timed_pages_q =
from e in no_select_timed_pages_q,
select: %{
page: e.pathname,
time_on_page: sum(e.duration),
visits: fragment("countIf(?)", e.transition)
}
"timed_pages"
|> with_cte("timed_pages", as: ^timed_pages_q)
|> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)
|> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)
|> select(
[t, i],
{
fragment("if(empty(?),?,?)", t.page, i.page, t.page),
(t.time_on_page + i.time_on_page) / (t.visits + i.visits)
}
)
else
from e in no_select_timed_pages_q,
select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)}
end
timed_pages_q
|> Plausible.ClickhouseRepo.all()
|> Map.new()
end
defp joins_table?(ecto_q, table) do
Enum.any?(
ecto_q.joins,

@ -801,6 +801,39 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|> json_response(200)
end
test "calculates time on page per unique transition within session", %{conn: conn, site: site} do
# ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐
# │ /a │ /b │ 100 │ s1 │
# │ /a │ /d │ 100 │ s2 │ <- these two get treated
# │ /a │ /d │ 0 │ s2 │ <- as single page transition
# └────┴────┴──────────────┴────┘
# so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100
s1 = @user_id
s2 = @user_id + 1
now = ~N[2021-01-01 00:00:00]
later = fn seconds -> NaiveDateTime.add(now, seconds) end
populate_stats(site, [
build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),
build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),
build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")
])
assert [
%{"name" => "/a", "time_on_page" => 100.0},
%{"name" => "/b", "time_on_page" => nil},
%{"name" => "/d", "time_on_page" => 0.0}
] =
conn
|> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
|> json_response(200)
end
test "calculates bounce rate and time on page for pages with imported data", %{
conn: conn,
site: site

@ -332,6 +332,43 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
|> Map.fetch!("top_stats")
|> Enum.find(&(&1["name"] == "Time on page"))
end
test "averages time on page over unique transitions across sessions", %{
conn: conn,
site: site
} do
# ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐
# │ /a │ /b │ 100 │ s1 │
# │ /a │ /d │ 100 │ s2 │ <- these two get treated
# │ /a │ /d │ 0 │ s2 │ <- as single page transition
# └────┴────┴──────────────┴────┘
# so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100
s1 = @user_id
s2 = @user_id + 1
now = ~N[2021-01-01 00:00:00]
later = fn seconds -> NaiveDateTime.add(now, seconds) end
populate_stats(site, [
build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),
build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),
build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),
build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")
])
filters = Jason.encode!(%{page: "/a"})
path = "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"
assert %{"name" => "Time on page", "value" => 100} ==
conn
|> get(path)
|> json_response(200)
|> Map.fetch!("top_stats")
|> Enum.find(&(&1["name"] == "Time on page"))
end
end
describe "GET /api/stats/top-stats - with imported data" do

@ -2,5 +2,6 @@
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:business_tier)
FunWithFlags.enable(:window_time_on_page)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
ExUnit.configure(exclude: [:slow])

Loading…
Cancel
Save