diff --git a/CHANGELOG.md b/CHANGELOG.md index be421dcac..9ace89bc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ### Added +- Group `operating_system_versions` by `operating_system` in Stats API breakdown +- Add `operating_system_versions.csv` into the CSV export - Display `Total visitors`, `Conversions`, and `CR` in the "Details" views of Countries, Regions and Cities (when filtering by a goal) - Add `conversion_rate` to Regions and Cities reports (when filtering by a goal) - Add the `conversion_rate` metric to Stats API Breakdown and Aggregate endpoints diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index bbb89091c..a0693310c 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -621,8 +621,9 @@ defmodule Plausible.Stats.Breakdown do defp do_group_by(q, "visit:os_version") do from( s in q, - group_by: s.operating_system_version, + group_by: [s.operating_system, s.operating_system_version], select_merge: %{ + os: fragment("if(empty(?), ?, ?)", s.operating_system, @not_set, s.operating_system), os_version: fragment( "if(empty(?), ?, ?)", @@ -695,26 +696,38 @@ defmodule Plausible.Stats.Breakdown do # the items in the given breakdown_results list page = 1 - # For browser_versions we need to fetch a lot more entries than the - # pagination limit. This is because many browsers can correspond to - # a single version number and we need to make sure that the results - # without goal filter will include all combinations of browsers and - # their versions that were present in the `breakdown_results` array. + # For browser/os versions we need to fetch a lot more entries than the + # pagination limit. This is because many entries can correspond to a + # single version number and we need to make sure that the results + # without goal filter will include all those combinations of browsers/os-s + # and their versions that were present in the `breakdown_results` array. {pagination_limit, find_match_fn} = - if property_atom == :browser_version do - pagination_limit = min(elem(pagination, 0) * 10, 3_000) + case property_atom do + :browser_version -> + pagination_limit = min(elem(pagination, 0) * 10, 3_000) - find_match_fn = fn total, conversion -> - total[:browser_version] == conversion[:browser_version] && - total[:browser] == conversion[:browser] - end + find_match_fn = fn total, conversion -> + total[:browser_version] == conversion[:browser_version] && + total[:browser] == conversion[:browser] + end - {pagination_limit, find_match_fn} - else - {elem(pagination, 0), - fn total, conversion -> - total[property_atom] == conversion[property_atom] - end} + {pagination_limit, find_match_fn} + + :os_version -> + pagination_limit = min(elem(pagination, 0) * 5, 3_000) + + find_match_fn = fn total, conversion -> + total[:os_version] == conversion[:os_version] && + total[:os] == conversion[:os] + end + + {pagination_limit, find_match_fn} + + _ -> + {elem(pagination, 0), + fn total, conversion -> + total[property_atom] == conversion[property_atom] + end} end pagination = {pagination_limit, page} diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index c3c0a6168..82e366773 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -1037,7 +1037,19 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{os_version: :name}) |> add_percentages(site, query) - json(conn, versions) + if params["csv"] do + if Map.has_key?(query.filters, "event:goal") do + versions + |> transform_keys(%{name: :version, os: :name, visitors: :conversions}) + |> to_csv([:name, :version, :conversions, :conversion_rate]) + else + versions + |> transform_keys(%{name: :version, os: :name}) + |> to_csv([:name, :version, :visitors]) + end + else + json(conn, versions) + end end def screen_sizes(conn, params) do diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 422b7d78d..7ab3f430b 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -157,6 +157,9 @@ defmodule PlausibleWeb.StatsController do ~c"browsers.csv" => fn -> Api.StatsController.browsers(conn, params) end, ~c"browser_versions.csv" => fn -> Api.StatsController.browser_versions(conn, params) end, ~c"operating_systems.csv" => fn -> Api.StatsController.operating_systems(conn, params) end, + ~c"operating_system_versions.csv" => fn -> + Api.StatsController.operating_system_versions(conn, params) + end, ~c"devices.csv" => fn -> Api.StatsController.screen_sizes(conn, params) end, ~c"conversions.csv" => fn -> Api.StatsController.conversions(conn, params) end, ~c"referrers.csv" => fn -> Api.StatsController.referrers(conn, params) end, diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_system_versions.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_system_versions.csv new file mode 100644 index 000000000..a81927435 --- /dev/null +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_system_versions.csv @@ -0,0 +1,2 @@ +name,version,conversions,conversion_rate +(not set),(not set),1,50.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv index cd761192e..89deb168d 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/operating_systems.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -(not set),1,25.0 +(not set),1,50.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/operating_system_versions.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/operating_system_versions.csv new file mode 100644 index 000000000..96f76c8b0 --- /dev/null +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/operating_system_versions.csv @@ -0,0 +1,2 @@ +name,version,visitors +(not set),(not set),1 diff --git a/test/plausible_web/controllers/CSVs/30d/operating_system_versions.csv b/test/plausible_web/controllers/CSVs/30d/operating_system_versions.csv new file mode 100644 index 000000000..d64000afa --- /dev/null +++ b/test/plausible_web/controllers/CSVs/30d/operating_system_versions.csv @@ -0,0 +1,3 @@ +name,version,visitors +(not set),(not set),2 +Mac,14,2 diff --git a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv index 9dc98c387..578344dba 100644 --- a/test/plausible_web/controllers/CSVs/30d/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/30d/operating_systems.csv @@ -1,2 +1,3 @@ name,visitors -(not set),4 +(not set),2 +Mac,2 diff --git a/test/plausible_web/controllers/CSVs/6m/operating_system_versions.csv b/test/plausible_web/controllers/CSVs/6m/operating_system_versions.csv new file mode 100644 index 000000000..2f10aaf69 --- /dev/null +++ b/test/plausible_web/controllers/CSVs/6m/operating_system_versions.csv @@ -0,0 +1,4 @@ +name,version,visitors +(not set),(not set),2 +Mac,14,2 +MacNoVersion,(not set),1 diff --git a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv index f574b0d4b..86a01a8ed 100644 --- a/test/plausible_web/controllers/CSVs/6m/operating_systems.csv +++ b/test/plausible_web/controllers/CSVs/6m/operating_systems.csv @@ -1,2 +1,4 @@ name,visitors -(not set),5 +(not set),2 +Mac,2 +MacNoVersion,1 diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 369992f00..1d3b5265c 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -532,32 +532,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:os_version", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, - operating_system_version: "10.5", - timestamp: ~N[2021-01-01 00:00:00] - ), - build(:pageview, - operating_system_version: "10.5", - timestamp: ~N[2021-01-01 00:25:00] - ), - build(:pageview, - operating_system_version: "10.6", - timestamp: ~N[2021-01-01 00:00:00] - ) + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system_version: "14"), + build(:pageview, operating_system: "Windows", operating_system_version: "11"), + build(:pageview, operating_system: "Windows", operating_system_version: "11") ]) conn = get(conn, "/api/v1/stats/breakdown", %{ "site_id" => site.domain, "period" => "day", - "date" => "2021-01-01", "property" => "visit:os_version" }) assert json_response(conn, 200) == %{ "results" => [ - %{"os_version" => "10.5", "visitors" => 2}, - %{"os_version" => "10.6", "visitors" => 1} + %{"os_version" => "14", "visitors" => 3, "os" => "Mac"}, + %{"os_version" => "11", "visitors" => 2, "os" => "Windows"}, + %{"os_version" => "14", "visitors" => 1, "os" => "(not set)"} ] } end diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index 43e1e1d81..d21aa884f 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -214,8 +214,8 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do ) assert json_response(conn, 200) == [ - %{"name" => "10.16", "visitors" => 2, "percentage" => 66.7}, - %{"name" => "10.15", "visitors" => 1, "percentage" => 33.3} + %{"name" => "10.16", "visitors" => 2, "percentage" => 66.7, "os" => "Mac"}, + %{"name" => "10.15", "visitors" => 1, "percentage" => 33.3, "os" => "Mac"} ] end end diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 0cb547379..963368d57 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -143,6 +143,7 @@ defmodule PlausibleWeb.StatsControllerTest do assert ~c"visitors.csv" in zip assert ~c"browsers.csv" in zip + assert ~c"browser_versions.csv" in zip assert ~c"cities.csv" in zip assert ~c"conversions.csv" in zip assert ~c"countries.csv" in zip @@ -150,6 +151,7 @@ defmodule PlausibleWeb.StatsControllerTest do assert ~c"entry_pages.csv" in zip assert ~c"exit_pages.csv" in zip assert ~c"operating_systems.csv" in zip + assert ~c"operating_system_versions.csv" in zip assert ~c"pages.csv" in zip assert ~c"regions.csv" in zip assert ~c"sources.csv" in zip @@ -287,6 +289,33 @@ defmodule PlausibleWeb.StatsControllerTest do [""] ] end + + test "exports operating system versions", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"), + build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"), + build(:pageview, operating_system: "Mac", operating_system_version: "13") + ]) + + conn = get(conn, "/#{site.domain}/export") + + assert response = response(conn, 200) + {:ok, zip} = :zip.unzip(response, [:memory]) + + {_filename, os_versions} = + Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end) + + assert parse_csv(os_versions) == [ + ["name", "version", "visitors"], + ["Mac", "14", "3"], + ["Ubuntu", "20.04", "2"], + ["Mac", "13", "1"], + [""] + ] + end end defp parse_csv(file_content) when is_binary(file_content) do @@ -412,14 +441,18 @@ defmodule PlausibleWeb.StatsControllerTest do timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1) |> NaiveDateTime.truncate(:second), browser: "Firefox", - browser_version: "120" + browser_version: "120", + operating_system: "Mac", + operating_system_version: "14" ), build(:pageview, timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1) |> NaiveDateTime.truncate(:second), country_code: "EE", browser: "Firefox", - browser_version: "120" + browser_version: "120", + operating_system: "Mac", + operating_system_version: "14" ), build(:pageview, timestamp: @@ -427,7 +460,8 @@ defmodule PlausibleWeb.StatsControllerTest do utm_campaign: "ads", country_code: "EE", referrer_source: "Google", - browser: "FirefoxNoVersion" + browser: "FirefoxNoVersion", + operating_system: "MacNoVersion" ), build(:event, timestamp: @@ -479,6 +513,51 @@ defmodule PlausibleWeb.StatsControllerTest do [""] ] end + + test "exports conversions and conversion rate for operating system versions", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, operating_system: "Mac", operating_system_version: "14"), + build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), + build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), + build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), + build(:event, + name: "Signup", + operating_system: "Ubuntu", + operating_system_version: "20.04" + ), + build(:event, + name: "Signup", + operating_system: "Ubuntu", + operating_system_version: "20.04" + ), + build(:event, + name: "Signup", + operating_system: "Lubuntu", + operating_system_version: "20.04" + ) + ]) + + insert(:goal, site: site, event_name: "Signup") + + conn = get(conn, "/#{site.domain}/export?filters=#{Jason.encode!(%{goal: "Signup"})}") + + assert response = response(conn, 200) + {:ok, zip} = :zip.unzip(response, [:memory]) + + {_filename, os_versions} = + Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end) + + assert parse_csv(os_versions) == [ + ["name", "version", "conversions", "conversion_rate"], + ["Mac", "14", "3", "75.0"], + ["Ubuntu", "20.04", "2", "100.0"], + ["Lubuntu", "20.04", "1", "100.0"], + [""] + ] + end end describe "GET /share/:domain?auth=:auth" do