From 4af7019011f5ef21928c98e9c8442786ccc61521 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 09:01:07 +0200 Subject: [PATCH] Ignore sessions without entry/exit pages when breaking down entry/exit pages (#3933) * Ignore sessions without entry/exit pages when breaking down entry/exit pages * Update stats controller tests to have more realistic test data (pageview followed by event) --- CHANGELOG.md | 1 + lib/plausible/stats/breakdown.ex | 4 +++ .../CSVs/30d-filter-goal/entry_pages.csv | 2 +- .../CSVs/30d-filter-goal/exit_pages.csv | 2 +- .../controllers/CSVs/30d/custom_props.csv | 2 +- .../controllers/CSVs/30d/entry_pages.csv | 2 +- .../controllers/CSVs/30d/exit_pages.csv | 2 +- .../controllers/CSVs/30d/pages.csv | 1 + .../controllers/CSVs/30d/referrers.csv | 2 +- .../controllers/CSVs/30d/sources.csv | 2 +- .../controllers/CSVs/30d/visitors.csv | 2 +- .../controllers/CSVs/6m/custom_props.csv | 2 +- .../controllers/CSVs/6m/entry_pages.csv | 2 +- .../controllers/CSVs/6m/exit_pages.csv | 2 +- .../controllers/CSVs/6m/pages.csv | 1 + .../controllers/CSVs/6m/referrers.csv | 2 +- .../controllers/CSVs/6m/sources.csv | 2 +- .../controllers/CSVs/6m/visitors.csv | 2 +- .../api/stats_controller/pages_test.exs | 36 +++++++++++++++++++ .../controllers/stats_controller_test.exs | 12 ++++++- 20 files changed, 68 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9263eb8d..a7742e03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ All notable changes to this project will be documented in this file. - Validate metric isn't queried multiple times ### Fixed +- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns - Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown - Fix `conversion_rate` metric in a `browser_versions` breakdown - Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 190d4cc4f..44d3fabf2 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -529,6 +529,8 @@ defmodule Plausible.Stats.Breakdown do defp do_group_by(q, "visit:entry_page") do from( s in q, + # Sessions without pageviews don't get entry_page assigned, hence they should get ignored + where: s.entry_page != "", group_by: s.entry_page, select_merge: %{entry_page: s.entry_page}, order_by: {:asc, s.entry_page} @@ -538,6 +540,8 @@ defmodule Plausible.Stats.Breakdown do defp do_group_by(q, "visit:exit_page") do from( s in q, + # Sessions without pageviews don't get entry_page assigned, hence they should get ignored + where: s.entry_page != "", group_by: s.exit_page, select_merge: %{exit_page: s.exit_page}, order_by: {:asc, s.exit_page} diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv index 2859d8c87..0f6f2cfa5 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,100.0 +/signup,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv index 2859d8c87..0f6f2cfa5 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,100.0 +/signup,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d/custom_props.csv b/test/plausible_web/controllers/CSVs/30d/custom_props.csv index 22ccc27a4..a0733de68 100644 --- a/test/plausible_web/controllers/CSVs/30d/custom_props.csv +++ b/test/plausible_web/controllers/CSVs/30d/custom_props.csv @@ -1,3 +1,3 @@ property,value,visitors,events,percentage variant,(none),3,4,75.0 -variant,A,1,1,25.0 +variant,A,1,2,25.0 diff --git a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv index a512fe4c3..59330a59c 100644 --- a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv @@ -1,3 +1,3 @@ name,unique_entrances,total_entrances,visit_duration /,3,3,20 -,1,1,0 +/signup,1,1,60 diff --git a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv index a48eaed31..036337efe 100644 --- a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv @@ -1,4 +1,4 @@ name,unique_exits,total_exits,exit_rate /,2,2,66.0 -,1,1, +/signup,1,1,100.0 /some-other-page,1,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv index 773bca0e3..e2990caa9 100644 --- a/test/plausible_web/controllers/CSVs/30d/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/pages.csv @@ -1,3 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,4,3,67, +/signup,1,1,0,60.0 /some-other-page,1,1,,60.0 diff --git a/test/plausible_web/controllers/CSVs/30d/referrers.csv b/test/plausible_web/controllers/CSVs/30d/referrers.csv index 12abdf9cf..b8a074551 100644 --- a/test/plausible_web/controllers/CSVs/30d/referrers.csv +++ b/test/plausible_web/controllers/CSVs/30d/referrers.csv @@ -1,2 +1,2 @@ name,visitors,bounce_rate,visit_duration -Direct / None,4,75,15 +Direct / None,4,50,30 diff --git a/test/plausible_web/controllers/CSVs/30d/sources.csv b/test/plausible_web/controllers/CSVs/30d/sources.csv index ac9093db0..f9d0b8c92 100644 --- a/test/plausible_web/controllers/CSVs/30d/sources.csv +++ b/test/plausible_web/controllers/CSVs/30d/sources.csv @@ -1,3 +1,3 @@ name,visitors,bounce_rate,visit_duration -Direct / None,3,100,0 +Direct / None,3,67,20 Google,1,0,60 diff --git a/test/plausible_web/controllers/CSVs/30d/visitors.csv b/test/plausible_web/controllers/CSVs/30d/visitors.csv index fcfe6aa24..6bac77976 100644 --- a/test/plausible_web/controllers/CSVs/30d/visitors.csv +++ b/test/plausible_web/controllers/CSVs/30d/visitors.csv @@ -28,5 +28,5 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration 2021-10-16,0,0,0,0.0,, 2021-10-17,0,0,0,0.0,, 2021-10-18,0,0,0,0.0,, -2021-10-19,2,1,2,0.5,100,0 +2021-10-19,2,2,2,1.0,50,30 2021-10-20,1,2,1,2.0,0,60 diff --git a/test/plausible_web/controllers/CSVs/6m/custom_props.csv b/test/plausible_web/controllers/CSVs/6m/custom_props.csv index 46a4e35d1..8c6ddd707 100644 --- a/test/plausible_web/controllers/CSVs/6m/custom_props.csv +++ b/test/plausible_web/controllers/CSVs/6m/custom_props.csv @@ -1,3 +1,3 @@ property,value,visitors,events,percentage variant,(none),4,5,80.0 -variant,A,1,1,20.0 +variant,A,1,2,20.0 diff --git a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv index 5ad9f3bf1..d0f98e949 100644 --- a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv @@ -1,3 +1,3 @@ name,unique_entrances,total_entrances,visit_duration /,4,4,15 -,1,1,0 +/signup,1,1,60 diff --git a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv index 44805add9..bb235645a 100644 --- a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv @@ -1,4 +1,4 @@ name,unique_exits,total_exits,exit_rate /,3,3,75.0 -,1,1, +/signup,1,1,100.0 /some-other-page,1,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv index a61e49779..805158544 100644 --- a/test/plausible_web/controllers/CSVs/6m/pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/pages.csv @@ -1,3 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,5,4,75, +/signup,1,1,0,60.0 /some-other-page,1,1,,60.0 diff --git a/test/plausible_web/controllers/CSVs/6m/referrers.csv b/test/plausible_web/controllers/CSVs/6m/referrers.csv index 85b5ad5e7..fc901f771 100644 --- a/test/plausible_web/controllers/CSVs/6m/referrers.csv +++ b/test/plausible_web/controllers/CSVs/6m/referrers.csv @@ -1,2 +1,2 @@ name,visitors,bounce_rate,visit_duration -Direct / None,5,80,12 +Direct / None,5,60,24 diff --git a/test/plausible_web/controllers/CSVs/6m/sources.csv b/test/plausible_web/controllers/CSVs/6m/sources.csv index a1d2a9fa4..259c3a23f 100644 --- a/test/plausible_web/controllers/CSVs/6m/sources.csv +++ b/test/plausible_web/controllers/CSVs/6m/sources.csv @@ -1,3 +1,3 @@ name,visitors,bounce_rate,visit_duration -Direct / None,3,100,0 +Direct / None,3,67,20 Google,2,50,30 diff --git a/test/plausible_web/controllers/CSVs/6m/visitors.csv b/test/plausible_web/controllers/CSVs/6m/visitors.csv index b6881dd9e..230cc9d82 100644 --- a/test/plausible_web/controllers/CSVs/6m/visitors.csv +++ b/test/plausible_web/controllers/CSVs/6m/visitors.csv @@ -4,4 +4,4 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration 2021-07-01,0,0,0,0.0,, 2021-08-01,0,0,0,0.0,, 2021-09-01,1,1,1,1.0,100,0 -2021-10-01,3,3,3,1.0,67,20 +2021-10-01,3,4,3,1.33,33,40 diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 976745ee5..5a7a0bd38 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -1324,6 +1324,24 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do } ] end + + test "ignores entry pages from sessions with only custom events", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:15:00], + pathname: "/" + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [] + end end describe "GET /api/stats/:domain/exit-pages" do @@ -1557,5 +1575,23 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do %{"name" => "/exit2", "visitors" => 1, "visits" => 1} ] end + + test "ignores exit pages from sessions with only custom events", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:15:00], + pathname: "/" + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [] + end end end diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 39d59c4cb..fd48e9cd9 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -285,7 +285,7 @@ defmodule PlausibleWeb.StatsControllerTest do ["2021-09-27", "0", "0", "0", "0.0", "", ""], ["2021-10-04", "0", "0", "0", "0.0", "", ""], ["2021-10-11", "0", "0", "0", "0.0", "", ""], - ["2021-10-18", "3", "3", "3", "1.0", "67", "20"], + ["2021-10-18", "3", "4", "3", "1.33", "33", "40"], [""] ] end @@ -469,7 +469,17 @@ defmodule PlausibleWeb.StatsControllerTest do browser: "FirefoxNoVersion", operating_system: "MacNoVersion" ), + build(:pageview, + user_id: 456, + timestamp: + Timex.shift(~N[2021-10-20 12:00:00], days: -1, minutes: -1) + |> NaiveDateTime.truncate(:second), + pathname: "/signup", + "meta.key": ["variant"], + "meta.value": ["A"] + ), build(:event, + user_id: 456, timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1) |> NaiveDateTime.truncate(:second), name: "Signup",