diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 694665dae..1e3c116f3 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -58,6 +58,14 @@ defmodule Plausible.Stats.Aggregate do end defp aggregate_time_on_page(site, query) do + if FunWithFlags.enabled?(:window_time_on_page) do + window_aggregate_time_on_page(site, query) + else + neighbor_aggregate_time_on_page(site, query) + end + end + + defp neighbor_aggregate_time_on_page(site, query) do q = from( e in base_event_query(site, %Query{ @@ -136,6 +144,50 @@ defmodule Plausible.Stats.Aggregate do %{time_on_page: time_on_page} end + defp window_aggregate_time_on_page(site, query) do + windowed_pages_q = + from e in base_event_query(site, %Query{ + query + | filters: Map.delete(query.filters, "event:page") + }), + select: %{ + next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), + next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), + timestamp: e.timestamp, + pathname: e.pathname, + session_id: e.session_id + }, + windows: [ + event_horizon: [ + partition_by: e.session_id, + order_by: e.timestamp, + frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") + ] + ] + + timed_page_transitions_q = + from e in Ecto.Query.subquery(windowed_pages_q), + group_by: [e.pathname, e.next_pathname, e.session_id], + where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname), + where: e.next_timestamp != 0, + select: %{ + pathname: e.pathname, + transition: e.next_pathname != e.pathname, + duration: sum(e.next_timestamp - e.timestamp) + } + + avg_time_per_page_transition_q = + from e in Ecto.Query.subquery(timed_page_transitions_q), + select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)}, + group_by: e.pathname + + time_on_page_q = + from e in Ecto.Query.subquery(avg_time_per_page_transition_q), + select: fragment("avg(ifNotFinite(?,NULL))", e.avg) + + %{time_on_page: ClickhouseRepo.one(time_on_page_q)} + end + @metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent] defp maybe_round_value({metric, nil}), do: {metric, 0} diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index dc04b706a..56e95ebfc 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -425,7 +425,7 @@ defmodule Plausible.Stats.Base do end # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity - defp dynamic_filter_condition(query, filter_key, db_field) do + def dynamic_filter_condition(query, filter_key, db_field) do case query && query.filters && query.filters[filter_key] do {:is, value} -> value = db_field_val(db_field, value) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 8287456bf..f86898c35 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -211,10 +211,18 @@ defmodule Plausible.Stats.Breakdown do end defp breakdown_time_on_page(_site, _query, []) do - [] + %{} end defp breakdown_time_on_page(site, query, pages) do + if FunWithFlags.enabled?(:window_time_on_page) do + window_breakdown_time_on_page(site, query, pages) + else + neighbor_breakdown_time_on_page(site, query, pages) + end + end + + defp neighbor_breakdown_time_on_page(site, query, pages) do q = from( e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])), @@ -294,6 +302,85 @@ defmodule Plausible.Stats.Breakdown do end end + defp window_breakdown_time_on_page(site, query, pages) do + import Ecto.Query + + windowed_pages_q = + from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])), + select: %{ + next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), + next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), + timestamp: e.timestamp, + pathname: e.pathname, + session_id: e.session_id + }, + windows: [ + event_horizon: [ + partition_by: e.session_id, + order_by: e.timestamp, + frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") + ] + ] + + timed_page_transitions_q = + from e in subquery(windowed_pages_q), + group_by: [e.pathname, e.next_pathname, e.session_id], + where: e.pathname in ^pages, + where: e.next_timestamp != 0, + select: %{ + pathname: e.pathname, + transition: e.next_pathname != e.pathname, + duration: sum(e.next_timestamp - e.timestamp) + } + + no_select_timed_pages_q = + from e in subquery(timed_page_transitions_q), + group_by: e.pathname + + timed_pages_q = + if query.include_imported do + # Imported page views have pre-calculated values + imported_timed_pages_q = + from i in "imported_pages", + group_by: i.page, + where: i.site_id == ^site.id, + where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, + where: i.page in ^pages, + select: %{ + page: i.page, + time_on_page: sum(i.time_on_page), + visits: sum(i.pageviews) - sum(i.exits) + } + + timed_pages_q = + from e in no_select_timed_pages_q, + select: %{ + page: e.pathname, + time_on_page: sum(e.duration), + visits: fragment("countIf(?)", e.transition) + } + + "timed_pages" + |> with_cte("timed_pages", as: ^timed_pages_q) + |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q) + |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page) + |> select( + [t, i], + { + fragment("if(empty(?),?,?)", t.page, i.page, t.page), + (t.time_on_page + i.time_on_page) / (t.visits + i.visits) + } + ) + else + from e in no_select_timed_pages_q, + select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)} + end + + timed_pages_q + |> Plausible.ClickhouseRepo.all() + |> Map.new() + end + defp joins_table?(ecto_q, table) do Enum.any?( ecto_q.joins, diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index fc28a2997..61c9c8d6f 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -801,6 +801,39 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do |> json_response(200) end + test "calculates time on page per unique transition within session", %{conn: conn, site: site} do + # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐ + # │ /a │ /b │ 100 │ s1 │ + # │ /a │ /d │ 100 │ s2 │ <- these two get treated + # │ /a │ /d │ 0 │ s2 │ <- as single page transition + # └────┴────┴──────────────┴────┘ + # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100 + + s1 = @user_id + s2 = @user_id + 1 + + now = ~N[2021-01-01 00:00:00] + later = fn seconds -> NaiveDateTime.add(now, seconds) end + + populate_stats(site, [ + build(:pageview, user_id: s1, timestamp: now, pathname: "/a"), + build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"), + build(:pageview, user_id: s2, timestamp: now, pathname: "/a"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d") + ]) + + assert [ + %{"name" => "/a", "time_on_page" => 100.0}, + %{"name" => "/b", "time_on_page" => nil}, + %{"name" => "/d", "time_on_page" => 0.0} + ] = + conn + |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true") + |> json_response(200) + end + test "calculates bounce rate and time on page for pages with imported data", %{ conn: conn, site: site diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs index 4a911d644..e9285aef4 100644 --- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs @@ -332,6 +332,43 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do |> Map.fetch!("top_stats") |> Enum.find(&(&1["name"] == "Time on page")) end + + test "averages time on page over unique transitions across sessions", %{ + conn: conn, + site: site + } do + # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐ + # │ /a │ /b │ 100 │ s1 │ + # │ /a │ /d │ 100 │ s2 │ <- these two get treated + # │ /a │ /d │ 0 │ s2 │ <- as single page transition + # └────┴────┴──────────────┴────┘ + # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100 + + s1 = @user_id + s2 = @user_id + 1 + + now = ~N[2021-01-01 00:00:00] + later = fn seconds -> NaiveDateTime.add(now, seconds) end + + populate_stats(site, [ + build(:pageview, user_id: s1, timestamp: now, pathname: "/a"), + build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"), + build(:pageview, user_id: s2, timestamp: now, pathname: "/a"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"), + build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d") + ]) + + filters = Jason.encode!(%{page: "/a"}) + path = "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}" + + assert %{"name" => "Time on page", "value" => 100} == + conn + |> get(path) + |> json_response(200) + |> Map.fetch!("top_stats") + |> Enum.find(&(&1["name"] == "Time on page")) + end end describe "GET /api/stats/top-stats - with imported data" do diff --git a/test/test_helper.exs b/test/test_helper.exs index 494caca6d..822e02d9e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,5 +2,6 @@ Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) Application.ensure_all_started(:double) FunWithFlags.enable(:business_tier) +FunWithFlags.enable(:window_time_on_page) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) ExUnit.configure(exclude: [:slow])