From 8a2df294f5a919725af78f795be7cbd61c5cf6bd Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Tue, 7 Feb 2023 12:47:54 +0100 Subject: [PATCH] Replace empty OS/screen size/Browser with `(not set)` in UI/API/exports (#2646) * Make Device section components aware of (not set) So that no extra sub-filters are possible when the unset top item is selected. * Support '(not set)' in breakdown/filters * Update expectations for export tests * Add extra tests for returning/filtering by '(not set)' * Add changelog entry * Remove ListReport conditional render * Prevent redundant sub-filters * Fix filter text rendering --------- Co-authored-by: Uku Taht --- CHANGELOG.md | 1 + assets/js/dashboard/filters.js | 23 ++++---- assets/js/dashboard/stats/devices/index.js | 57 +++++++++++-------- lib/plausible/stats/base.ex | 2 + lib/plausible/stats/breakdown.ex | 29 ++++++++-- .../CSVs/30d-filter-goal/browsers.csv | 2 +- .../CSVs/30d-filter-goal/devices.csv | 2 +- .../30d-filter-goal/operating_systems.csv | 2 +- .../CSVs/30d-filter-path/browsers.csv | 2 +- .../CSVs/30d-filter-path/devices.csv | 2 +- .../30d-filter-path/operating_systems.csv | 2 +- .../controllers/CSVs/30d/browsers.csv | 2 +- .../controllers/CSVs/30d/devices.csv | 2 +- .../CSVs/30d/operating_systems.csv | 2 +- .../controllers/CSVs/6m/browsers.csv | 2 +- .../controllers/CSVs/6m/devices.csv | 2 +- .../controllers/CSVs/6m/operating_systems.csv | 2 +- .../api/stats_controller/browsers_test.exs | 33 +++++++++++ .../operating_systems_test.exs | 27 +++++++++ .../stats_controller/screen_sizes_test.exs | 27 +++++++++ 20 files changed, 173 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adc00dd91..a237a87ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. - 'Last updated X seconds ago' info to 'current visitors' tooltips ### Fixed +- Empty values for Screen Size, OS and Browser are uniformly replaced with "(not set)" - Fix [more pageviews with session prop filter than with no filters](https://github.com/plausible/analytics/issues/1666) - Cascade delete sent_renewal_notifications table when user is deleted plausible/analytics#2549 - Show appropriate top-stat metric labels on the realtime dashboard when filtering by a goal diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 98413e60a..8c21a24c9 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -18,8 +18,8 @@ function removeFilter(key, history, query) { [key]: false } if (key === 'country') { newOpts.country_name = false } - if (key === 'region') { newOpts.region_name = false } - if (key === 'city') { newOpts.city_name = false } + if (key === 'region') { newOpts.region_name = false } + if (key === 'city') { newOpts.city_name = false } navigateToQuery( history, @@ -50,11 +50,14 @@ function filterText(key, rawValue, query) { return <>{eventName}.{metaKey} {toFilterType(metaValue)} {valueWithoutPrefix(metaValue)} } if (key === "browser_version") { - const browserName = query.filters.browser ? query.filters.browser : 'Browser' + const isNotSet = query.filters.browser === '(not set)' || (!query.filters.browser) + const browserName = isNotSet ? 'Browser' : query.filters.browser return <>{browserName}.Version {type} {value} } if (key === "os_version") { - const osName = query.filters.os ? query.filters.os : 'OS' + const isNotSet = query.filters.os === '(not set)' || (!query.filters.os) + const osName = isNotSet ? 'Operating System' : query.filters.os + console.info('osname', osName) return <>{osName}.Version {type} {value} } if (key === "country") { @@ -92,7 +95,7 @@ function renderDropdownFilter(site, history, [key, value], query) { title={`Edit filter: ${formattedFilters[key]}`} to={{ pathname: `/${encodeURIComponent(site.domain)}/filter/${filterGroupForFilter(key)}`, search: window.location.search }} className="group flex w-full justify-between items-center" - style={{width: 'calc(100% - 1.5rem)'}} + style={{ width: 'calc(100% - 1.5rem)' }} > {filterText(key, value, query)} @@ -123,7 +126,7 @@ function filterDropdownOption(site, option) { ) } -function DropdownContent({history, site, query, wrapped}) { +function DropdownContent({ history, site, query, wrapped }) { const [addingFilter, setAddingFilter] = useState(false); if (wrapped === 0 || addingFilter) { @@ -191,7 +194,7 @@ class Filters extends React.Component { } handleKeyup(e) { - const {query, history} = this.props + const { query, history } = this.props if (e.ctrlKey || e.metaKey || e.altKey) return @@ -201,7 +204,7 @@ class Filters extends React.Component { } handleResize() { - this.setState({ viewport: window.innerWidth || 639}); + this.setState({ viewport: window.innerWidth || 639 }); } // Checks if the filter container is wrapping items @@ -325,8 +328,8 @@ class Filters extends React.Component { render() { return ( <> - { this.renderFilterList() } - { this.renderDropDown() } + {this.renderFilterList()} + {this.renderDropDown()} ) } diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js index 214e7e3db..451caed53 100644 --- a/assets/js/dashboard/stats/devices/index.js +++ b/assets/js/dashboard/stats/devices/index.js @@ -5,7 +5,7 @@ import ListReport from '../reports/list' import * as api from '../../api' import * as url from '../../util/url' -function Browsers({query, site}) { +function Browsers({ query, site }) { function fetchData() { return api.get(url.apiPath(site, '/browsers'), query) } @@ -13,29 +13,34 @@ function Browsers({query, site}) { return ( ) } -function BrowserVersions({query, site}) { +function BrowserVersions({ query, site }) { function fetchData() { return api.get(url.apiPath(site, '/browser-versions'), query) } + const isNotSet = query.filters.browser === '(not set)' + const browserName = isNotSet ? 'Browser' : query.filters.browser + const filter = isNotSet ? {} : { browser_version: 'name' } + return ( ) + } -function OperatingSystems({query, site}) { +function OperatingSystems({ query, site }) { function fetchData() { return api.get(url.apiPath(site, '/operating-systems'), query) } @@ -43,29 +48,34 @@ function OperatingSystems({query, site}) { return ( ) } -function OperatingSystemVersions({query, site}) { +function OperatingSystemVersions({ query, site }) { function fetchData() { return api.get(url.apiPath(site, '/operating-system-versions'), query) } + const isNotSet = query.filters.os === '(not set)' + const osName = isNotSet ? 'Operating System' : query.filters.os + const filter = isNotSet ? {} : { os_version: 'name' } + return ( ) + } -function ScreenSizes({query, site}) { +function ScreenSizes({ query, site }) { function fetchData() { return api.get(url.apiPath(site, '/screen-sizes'), query) } @@ -81,7 +91,7 @@ function ScreenSizes({query, site}) { return ( + ) } else if (screenSize === 'Tablet') { return ( - + ) } else if (screenSize === 'Laptop') { return ( - + ) } else if (screenSize === 'Desktop') { return ( - + ) + } else if (screenSize === '(not set)') { + return null } } export default class Devices extends React.Component { constructor(props) { super(props) - this.tabKey = `deviceTab__${ props.site.domain}` + this.tabKey = `deviceTab__${props.site.domain}` const storedTab = storage.getItem(this.tabKey) this.state = { mode: storedTab || 'size' } } - setMode(mode) { return () => { storage.setItem(this.tabKey, mode) - this.setState({mode}) + this.setState({ mode }) } } @@ -189,12 +200,12 @@ export default class Devices extends React.Component {

Devices

- { this.renderPill('Size', 'size') } - { this.renderPill('Browser', 'browser') } - { this.renderPill('OS', 'os') } + {this.renderPill('Size', 'size')} + {this.renderPill('Browser', 'browser')} + {this.renderPill('OS', 'os')}
- { this.renderContent() } + {this.renderContent()} ) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 24de1aeb9..c27b102a5 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -6,6 +6,7 @@ defmodule Plausible.Stats.Base do # Ecto typespec has not been updated for this PR: https://github.com/elixir-ecto/ecto/pull/3592 @dialyzer {:nowarn_function, add_sample_hint: 2} @no_ref "Direct / None" + @not_set "(not set)" def base_event_query(site, query) do events_q = query_events(site, query) @@ -391,6 +392,7 @@ defmodule Plausible.Stats.Base do defp db_prop_val(:utm_campaign, @no_ref), do: "" defp db_prop_val(:utm_content, @no_ref), do: "" defp db_prop_val(:utm_term, @no_ref), do: "" + defp db_prop_val(_, @not_set), do: "" defp db_prop_val(_, val), do: val def utc_boundaries(%Query{period: "realtime"}, _timezone) do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 10e4e7385..77842528e 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -5,6 +5,7 @@ defmodule Plausible.Stats.Breakdown do alias Plausible.Stats.Query alias Plausible.Goals @no_ref "Direct / None" + @not_set "(not set)" @event_metrics [:visitors, :pageviews, :events] @session_metrics [:visits, :bounce_rate, :visit_duration] @@ -512,7 +513,9 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.screen_size, - select_merge: %{device: s.screen_size}, + select_merge: %{ + device: fragment("if(empty(?), ?, ?)", s.screen_size, @not_set, s.screen_size) + }, order_by: {:asc, s.screen_size} ) end @@ -521,7 +524,10 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system, - select_merge: %{operating_system: s.operating_system}, + select_merge: %{ + operating_system: + fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system) + }, order_by: {:asc, s.operating_system} ) end @@ -530,7 +536,15 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system_version, - select_merge: %{os_version: s.operating_system_version}, + select_merge: %{ + os_version: + fragment( + "if(empty(?), ?, ?)", + s.operating_system_version, + @not_set, + s.operating_system_version + ) + }, order_by: {:asc, s.operating_system_version} ) end @@ -539,7 +553,9 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.browser, - select_merge: %{browser: s.browser}, + select_merge: %{ + browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser) + }, order_by: {:asc, s.browser} ) end @@ -548,7 +564,10 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.browser_version, - select_merge: %{browser_version: s.browser_version}, + select_merge: %{ + browser_version: + fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version) + }, order_by: {:asc, s.browser_version} ) end diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/browsers.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/browsers.csv index 292175e17..89deb168d 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/browsers.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/browsers.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,50.0 +(not set),1,50.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/devices.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/devices.csv index 64f760da4..cd761192e 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/devices.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/devices.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,25.0 +(not set),1,25.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv index 64f760da4..cd761192e 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,25.0 +(not set),1,25.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/browsers.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/browsers.csv index 7b062ae5b..d7aedd4cc 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/browsers.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/browsers.csv @@ -1,2 +1,2 @@ name,visitors -,1 +(not set),1 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/devices.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/devices.csv index 7b062ae5b..d7aedd4cc 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/devices.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/devices.csv @@ -1,2 +1,2 @@ name,visitors -,1 +(not set),1 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/operating_systems.csv index 7b062ae5b..d7aedd4cc 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/operating_systems.csv @@ -1,2 +1,2 @@ name,visitors -,1 +(not set),1 diff --git a/test/plausible_web/controllers/CSVs/30d/browsers.csv b/test/plausible_web/controllers/CSVs/30d/browsers.csv index b09641f47..0065f7b3d 100644 --- a/test/plausible_web/controllers/CSVs/30d/browsers.csv +++ b/test/plausible_web/controllers/CSVs/30d/browsers.csv @@ -1,3 +1,3 @@ name,visitors ABrowserName,2 -,2 +(not set),2 diff --git a/test/plausible_web/controllers/CSVs/30d/devices.csv b/test/plausible_web/controllers/CSVs/30d/devices.csv index 259a48cdd..9dc98c387 100644 --- a/test/plausible_web/controllers/CSVs/30d/devices.csv +++ b/test/plausible_web/controllers/CSVs/30d/devices.csv @@ -1,2 +1,2 @@ name,visitors -,4 +(not set),4 diff --git a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv index 259a48cdd..9dc98c387 100644 --- a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv @@ -1,2 +1,2 @@ name,visitors -,4 +(not set),4 diff --git a/test/plausible_web/controllers/CSVs/6m/browsers.csv b/test/plausible_web/controllers/CSVs/6m/browsers.csv index f2719de78..10141774c 100644 --- a/test/plausible_web/controllers/CSVs/6m/browsers.csv +++ b/test/plausible_web/controllers/CSVs/6m/browsers.csv @@ -1,3 +1,3 @@ name,visitors ABrowserName,3 -,2 +(not set),2 diff --git a/test/plausible_web/controllers/CSVs/6m/devices.csv b/test/plausible_web/controllers/CSVs/6m/devices.csv index 6d03d7369..f574b0d4b 100644 --- a/test/plausible_web/controllers/CSVs/6m/devices.csv +++ b/test/plausible_web/controllers/CSVs/6m/devices.csv @@ -1,2 +1,2 @@ name,visitors -,5 +(not set),5 diff --git a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv index 6d03d7369..f574b0d4b 100644 --- a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv @@ -1,2 +1,2 @@ name,visitors -,5 +(not set),5 diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index ad318720f..c5f670a83 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -129,6 +129,21 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do %{"name" => "Firefox", "visitors" => 1, "percentage" => 33} ] end + + test "returns (not set) when appropriate", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + user_id: 123, + browser: "" + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 100} + ] + end end describe "GET /api/stats/:domain/browser-versions" do @@ -155,5 +170,23 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do %{"name" => "77.0", "visitors" => 1, "percentage" => 33} ] end + + test "returns results for (not set)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "", browser_version: "") + ]) + + filters = Jason.encode!(%{browser: "(not set)"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 100} + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index b760b6bc2..2b73636d7 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -19,6 +19,33 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do ] end + test "returns (not set) when appropriate", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + operating_system: "" + ), + build(:pageview, + operating_system: "Linux" + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 50}, + %{"name" => "Linux", "visitors" => 1, "percentage" => 50} + ] + + filters = Jason.encode!(%{os: "(not set)"}) + + conn = + get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 100} + ] + end + test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, user_id: 1, operating_system: "Mac"), diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 7b3ecab52..22f0d837b 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -19,6 +19,33 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do ] end + test "returns (not set) when appropriate", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + screen_size: "" + ), + build(:pageview, + screen_size: "Desktop" + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 50}, + %{"name" => "Desktop", "visitors" => 1, "percentage" => 50} + ] + + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day") + + filters = Jason.encode!(%{screen: "(not set)"}) + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "(not set)", "visitors" => 1, "percentage" => 100} + ] + end + test "returns screen sizes with :is filter on custom pageview props", %{ conn: conn, site: site