From f576fa2a2c6a9296f01b4a5169c9301d2686f4c3 Mon Sep 17 00:00:00 2001 From: Matt Colligan <44536113+m-col@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:20:39 +0000 Subject: [PATCH] Improvements to CSV export (#1427) * Add details=True to export API parameters This makes the ZIP export add `%{"details" => "True"}` to the query's `params` when fetching data internally for packaging in the ZIP. This adds bounce_rate and time_on_page to the data in pages.csv, and bounce_rate and visit_duration to sources.csv. * Make API return data with consistent names Some of the data types returned via the JSON or CSV API use inconsistent naming, and some have redundant name changes (i.e. count -> visitors -> count). This makes these all consistent and removes the redundancy. This addresses #1426, fixes some of the CSV headers, and unifies the JSON and CSV return data labels. * Update changelog * Test should use Timex.shift, not relative time * Return full country names in CSV export This also replaces the " character with ' in two country names, as those are the characters used in the names, yielding a more predictable and 'correct' output. * Fetch CSV exported data concurrently * Use spinner to indicate when export has started * Use 300 as default number of brekadown entries for export Higher numbers (e.g. 1000) seem to cause clickhouse errors when there many pages to request. It is unclear what is causing the error, as clickhouse returns an "unknown" error code and an empty error message. --- CHANGELOG.md | 1 + assets/js/dashboard/stats/bar.js | 12 +- .../js/dashboard/stats/conversions/index.js | 4 +- .../stats/conversions/prop-breakdown.js | 4 +- assets/js/dashboard/stats/devices/browsers.js | 4 +- assets/js/dashboard/stats/devices/index.js | 4 +- .../stats/devices/operating-systems.js | 4 +- .../js/dashboard/stats/locations/countries.js | 4 +- assets/js/dashboard/stats/locations/map.js | 4 +- assets/js/dashboard/stats/modals/countries.js | 2 +- .../js/dashboard/stats/modals/entry-pages.js | 4 +- .../js/dashboard/stats/modals/exit-pages.js | 4 +- .../dashboard/stats/modals/google-keywords.js | 2 +- assets/js/dashboard/stats/modals/pages.js | 2 +- .../stats/modals/referrer-drilldown.js | 4 +- assets/js/dashboard/stats/modals/sources.js | 2 +- .../js/dashboard/stats/pages/entry-pages.js | 5 +- assets/js/dashboard/stats/pages/exit-pages.js | 5 +- assets/js/dashboard/stats/pages/pages.js | 4 +- .../dashboard/stats/sources/referrer-list.js | 4 +- .../dashboard/stats/sources/search-terms.js | 4 +- .../js/dashboard/stats/sources/source-list.js | 8 +- assets/js/dashboard/stats/visitor-graph.js | 33 +++- lib/plausible/stats/breakdown.ex | 4 + lib/plausible/stats/countries.ex | 4 +- .../controllers/api/stats_controller.ex | 94 +++++----- .../controllers/stats_controller.ex | 35 ++-- .../api/stats_controller/browsers_test.exs | 10 +- .../api/stats_controller/conversions_test.exs | 79 ++++++--- .../api/stats_controller/countries_test.exs | 8 +- .../operating_systems_test.exs | 10 +- .../api/stats_controller/pages_test.exs | 48 +++--- .../stats_controller/screen_sizes_test.exs | 6 +- .../api/stats_controller/sources_test.exs | 42 ++--- .../controllers/stats_controller_test.exs | 160 +++++++++--------- 35 files changed, 353 insertions(+), 271 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55e754046..46a5ece9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added - Data exported via the download button will contain CSV data for all visible graps in a zip file. +- JSON API has renamed some metric items to be more clear and match the CSV API output. ### Fixed - Fixes database error when pathname contains a question mark diff --git a/assets/js/dashboard/stats/bar.js b/assets/js/dashboard/stats/bar.js index f1585b3f2..1acb6f977 100644 --- a/assets/js/dashboard/stats/bar.js +++ b/assets/js/dashboard/stats/bar.js @@ -1,17 +1,17 @@ import React from 'react'; -function barWidth(count, all) { - let maxVal = all[0].count; +function barWidth(count, all, plot) { + let maxVal = all[0][plot]; - for (const entry of all) { - if (entry.count > maxVal) maxVal = entry.count + for (const val of all) { + if (val > maxVal) maxVal = val[plot] } return count / maxVal * 100 } -export default function Bar({count, all, bg, maxWidthDeduction, children}) { - const width = barWidth(count, all) +export default function Bar({count, all, bg, maxWidthDeduction, children, plot = "visitors"}) { + const width = barWidth(count, all, plot) return (
- {numberFormatter(goal.count)} + {numberFormatter(goal.visitors)} {viewport > MOBILE_UPPER_WIDTH && {numberFormatter(goal.total_count)}} {goal.conversion_rate}%
diff --git a/assets/js/dashboard/stats/conversions/prop-breakdown.js b/assets/js/dashboard/stats/conversions/prop-breakdown.js index 7811df43e..cab4e4a5a 100644 --- a/assets/js/dashboard/stats/conversions/prop-breakdown.js +++ b/assets/js/dashboard/stats/conversions/prop-breakdown.js @@ -115,7 +115,7 @@ export default class PropertyBreakdown extends React.Component { return (
- {numberFormatter(value.count)} + {numberFormatter(value.visitors)} { viewport > MOBILE_UPPER_WIDTH ? ( diff --git a/assets/js/dashboard/stats/devices/browsers.js b/assets/js/dashboard/stats/devices/browsers.js index 256820767..6dd907f4a 100644 --- a/assets/js/dashboard/stats/devices/browsers.js +++ b/assets/js/dashboard/stats/devices/browsers.js @@ -68,7 +68,7 @@ export default class Browsers extends React.Component { return (
- {numberFormatter(browser.count)} ({browser.percentage}%) + {numberFormatter(browser.visitors)} ({browser.percentage}%) {this.showConversionRate() && {numberFormatter(browser.conversion_rate)}%}
diff --git a/assets/js/dashboard/stats/devices/index.js b/assets/js/dashboard/stats/devices/index.js index e2905746f..1447af65e 100644 --- a/assets/js/dashboard/stats/devices/index.js +++ b/assets/js/dashboard/stats/devices/index.js @@ -83,7 +83,7 @@ class ScreenSizes extends React.Component { return (
- {numberFormatter(size.count)} ({size.percentage}%) + {numberFormatter(size.visitors)} ({size.percentage}%) {this.showConversionRate() && {numberFormatter(size.conversion_rate)}%}
diff --git a/assets/js/dashboard/stats/devices/operating-systems.js b/assets/js/dashboard/stats/devices/operating-systems.js index 5603c1a14..9a9d50470 100644 --- a/assets/js/dashboard/stats/devices/operating-systems.js +++ b/assets/js/dashboard/stats/devices/operating-systems.js @@ -56,7 +56,7 @@ export default class OperatingSystems extends React.Component { key={os.name} > - {numberFormatter(os.count)} ({os.percentage}%) + {numberFormatter(os.visitors)} ({os.percentage}%) {this.showConversionRate() && {numberFormatter(os.conversion_rate)}%}
) diff --git a/assets/js/dashboard/stats/locations/countries.js b/assets/js/dashboard/stats/locations/countries.js index 8e8e77137..b2c2180bd 100644 --- a/assets/js/dashboard/stats/locations/countries.js +++ b/assets/js/dashboard/stats/locations/countries.js @@ -59,7 +59,7 @@ export default class Countries extends React.Component { key={country.name} > - {numberFormatter(country.count)} + {numberFormatter(country.visitors)} {this.showConversionRate() && {numberFormatter(country.conversion_rate)}%}
) diff --git a/assets/js/dashboard/stats/locations/map.js b/assets/js/dashboard/stats/locations/map.js index 8dc82d49c..4839df654 100644 --- a/assets/js/dashboard/stats/locations/map.js +++ b/assets/js/dashboard/stats/locations/map.js @@ -49,7 +49,7 @@ class CountriesMap extends React.Component { getDataset() { var dataset = {}; - var onlyValues = this.state.countries.map(function(obj){ return obj.count }); + var onlyValues = this.state.countries.map(function(obj){ return obj.visitors }); var maxValue = Math.max.apply(null, onlyValues); var paletteScale = d3.scale.linear() @@ -60,7 +60,7 @@ class CountriesMap extends React.Component { ]); this.state.countries.forEach(function(item){ - dataset[item.name] = {numberOfThings: item.count, fillColor: paletteScale(item.count)}; + dataset[item.name] = {numberOfThings: item.visitors, fillColor: paletteScale(item.visitors)}; }); return dataset diff --git a/assets/js/dashboard/stats/modals/countries.js b/assets/js/dashboard/stats/modals/countries.js index 1819e5c2d..cce890f54 100644 --- a/assets/js/dashboard/stats/modals/countries.js +++ b/assets/js/dashboard/stats/modals/countries.js @@ -58,7 +58,7 @@ class CountriesModal extends React.Component { {this.showConversionRate() && {country.total_visitors} } - {numberFormatter(country.count)} {!this.showConversionRate() && ({country.percentage}%)} + {numberFormatter(country.visitors)} {!this.showConversionRate() && ({country.percentage}%)} {this.showConversionRate() && {country.conversion_rate}% } diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.js index ddb6b3a51..198c32f36 100644 --- a/assets/js/dashboard/stats/modals/entry-pages.js +++ b/assets/js/dashboard/stats/modals/entry-pages.js @@ -90,8 +90,8 @@ class EntryPagesModal extends React.Component { {this.showConversionRate() && {numberFormatter(page.total_visitors)}} - {numberFormatter(page.count)} - {this.showExtra() && {numberFormatter(page.entries)}} + {numberFormatter(page.unique_entrances)} + {this.showExtra() && {numberFormatter(page.total_entrances)}} {this.showExtra() && {durationFormatter(page.visit_duration)}} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%} diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js index d0eb0961b..21b895214 100644 --- a/assets/js/dashboard/stats/modals/exit-pages.js +++ b/assets/js/dashboard/stats/modals/exit-pages.js @@ -72,8 +72,8 @@ class ExitPagesModal extends React.Component { {page.name} {this.showConversionRate() && {numberFormatter(page.total_visitors)}} - {numberFormatter(page.count)} - {this.showExtra() && {numberFormatter(page.exits)}} + {numberFormatter(page.unique_exits)} + {this.showExtra() && {numberFormatter(page.total_exits)}} {this.showExtra() && {this.formatPercentage(page.exit_rate)}} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%} diff --git a/assets/js/dashboard/stats/modals/google-keywords.js b/assets/js/dashboard/stats/modals/google-keywords.js index 0dcf4b624..b2ac56957 100644 --- a/assets/js/dashboard/stats/modals/google-keywords.js +++ b/assets/js/dashboard/stats/modals/google-keywords.js @@ -44,7 +44,7 @@ class GoogleKeywordsModal extends React.Component { {term.name} - {numberFormatter(term.count)} + {numberFormatter(term.visitors)} ) diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js index 7208974da..f243298b5 100644 --- a/assets/js/dashboard/stats/modals/pages.js +++ b/assets/js/dashboard/stats/modals/pages.js @@ -67,7 +67,7 @@ class PagesModal extends React.Component { {page.name} {this.showConversionRate() && {page.total_visitors} } - {numberFormatter(page.count)} + {numberFormatter(page.visitors)} {this.showPageviews() && {numberFormatter(page.pageviews)} } {this.showExtra() && {this.formatBounceRate(page)} } {this.showExtra() && {timeOnPage} } diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index e97a25ffe..a38a55aa9 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -124,7 +124,7 @@ class ReferrerDrilldownModal extends React.Component {
{this.showConversionRate() && {numberFormatter(referrer.total_visitors)} } - {numberFormatter(referrer.count)} + {numberFormatter(referrer.visitors)} {this.showExtra() && {this.formatBounceRate(referrer)} } {this.showExtra() && {this.formatDuration(referrer)} } {this.showConversionRate() && {referrer.conversion_rate}% } @@ -137,7 +137,7 @@ class ReferrerDrilldownModal extends React.Component { { this.renderReferrerName(referrer) } {this.showConversionRate() && {numberFormatter(referrer.total_visitors)} } - {numberFormatter(referrer.count)} + {numberFormatter(referrer.visitors)} {this.showExtra() && {this.formatBounceRate(referrer)} } {this.showExtra() && {this.formatDuration(referrer)} } {this.showConversionRate() && {referrer.conversion_rate}% } diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.js index d74aa9b3c..2005422ca 100644 --- a/assets/js/dashboard/stats/modals/sources.js +++ b/assets/js/dashboard/stats/modals/sources.js @@ -97,7 +97,7 @@ class SourcesModal extends React.Component { { source.name } {this.showConversionRate() && {numberFormatter(source.total_visitors)} } - {numberFormatter(source.count)} + {numberFormatter(source.visitors)} {this.showExtra() && {this.formatBounceRate(source)} } {this.showExtra() && {this.formatDuration(source)} } {this.showConversionRate() && {source.conversion_rate}% } diff --git a/assets/js/dashboard/stats/pages/entry-pages.js b/assets/js/dashboard/stats/pages/entry-pages.js index 13414c29c..58689fc31 100644 --- a/assets/js/dashboard/stats/pages/entry-pages.js +++ b/assets/js/dashboard/stats/pages/entry-pages.js @@ -57,8 +57,9 @@ export default class EntryPages extends React.Component { return (
@@ -79,7 +80,7 @@ export default class EntryPages extends React.Component { - {numberFormatter(page.count)} + {numberFormatter(page.unique_entrances)} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%}
) diff --git a/assets/js/dashboard/stats/pages/exit-pages.js b/assets/js/dashboard/stats/pages/exit-pages.js index ba8970a59..b7d2e3aaf 100644 --- a/assets/js/dashboard/stats/pages/exit-pages.js +++ b/assets/js/dashboard/stats/pages/exit-pages.js @@ -57,8 +57,9 @@ export default class ExitPages extends React.Component { return (
@@ -79,7 +80,7 @@ export default class ExitPages extends React.Component { - {numberFormatter(page.count)} + {numberFormatter(page.unique_exits)} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%}
) diff --git a/assets/js/dashboard/stats/pages/pages.js b/assets/js/dashboard/stats/pages/pages.js index d9c153516..438dfdd0b 100644 --- a/assets/js/dashboard/stats/pages/pages.js +++ b/assets/js/dashboard/stats/pages/pages.js @@ -60,7 +60,7 @@ export default class Visits extends React.Component { key={page.name} > - {numberFormatter(page.count)} + {numberFormatter(page.visitors)} {this.showConversionRate() && {numberFormatter(page.conversion_rate)}%}
) diff --git a/assets/js/dashboard/stats/sources/referrer-list.js b/assets/js/dashboard/stats/sources/referrer-list.js index 5e32a5579..f4bc65c84 100644 --- a/assets/js/dashboard/stats/sources/referrer-list.js +++ b/assets/js/dashboard/stats/sources/referrer-list.js @@ -78,7 +78,7 @@ export default class Referrers extends React.Component { return (
- {numberFormatter(referrer.count)} + {numberFormatter(referrer.visitors)} {this.showConversionRate() && {referrer.conversion_rate}%}
) diff --git a/assets/js/dashboard/stats/sources/search-terms.js b/assets/js/dashboard/stats/sources/search-terms.js index 6c5c989ac..5d5f89954 100644 --- a/assets/js/dashboard/stats/sources/search-terms.js +++ b/assets/js/dashboard/stats/sources/search-terms.js @@ -37,7 +37,7 @@ export default class SearchTerms extends React.Component { return (
- {numberFormatter(term.count)} + {numberFormatter(term.visitors)}
) } diff --git a/assets/js/dashboard/stats/sources/source-list.js b/assets/js/dashboard/stats/sources/source-list.js index a8a02e827..3c89dbde0 100644 --- a/assets/js/dashboard/stats/sources/source-list.js +++ b/assets/js/dashboard/stats/sources/source-list.js @@ -52,7 +52,7 @@ class AllSources extends React.Component { key={referrer.name} > - {numberFormatter(referrer.count)} + {numberFormatter(referrer.visitors)} {this.showConversionRate() && {referrer.conversion_rate}%} ) @@ -184,7 +184,7 @@ class UTMSources extends React.Component { key={referrer.name} > - {numberFormatter(referrer.count)} + {numberFormatter(referrer.visitors)} {this.showConversionRate() && {referrer.conversion_rate}%} ) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index c359d7817..f83a4a421 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -101,6 +101,9 @@ class LineGraph extends React.Component { super(props); this.regenerateChart = this.regenerateChart.bind(this); this.updateWindowDimensions = this.updateWindowDimensions.bind(this); + this.state = { + exported: false + }; } regenerateChart() { @@ -301,15 +304,33 @@ class LineGraph extends React.Component { return stats } + downloadSpinner() { + this.setState({exported: true}); + setTimeout( + () => this.setState({exported: false}), + 4000 + ); + } + downloadLink() { if (this.props.query.period !== 'realtime') { - const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}` - return ( - - - - ) + if (this.state.exported) { + return ( + + + + + ) + } else { + const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}` + + return ( + + + + ) + } } } diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index b45c80d8f..162fe8fcc 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -206,6 +206,10 @@ defmodule Plausible.Stats.Breakdown do |> ClickhouseRepo.all() end + defp breakdown_time_on_page(_site, _query, []) do + [] + end + defp breakdown_time_on_page(site, query, pages) do q = from( diff --git a/lib/plausible/stats/countries.ex b/lib/plausible/stats/countries.ex index 76aae0837..86eb6d074 100644 --- a/lib/plausible/stats/countries.ex +++ b/lib/plausible/stats/countries.ex @@ -53,7 +53,7 @@ defmodule Plausible.Stats.CountryName do "CD" => "Congo, Democratic Republic", "CK" => "Cook Islands", "CR" => "Costa Rica", - "CI" => "Cote D\"Ivoire", + "CI" => "Cote D'Ivoire", "HR" => "Croatia", "CU" => "Cuba", "CY" => "Cyprus", @@ -118,7 +118,7 @@ defmodule Plausible.Stats.CountryName do "KR" => "Korea", "KW" => "Kuwait", "KG" => "Kyrgyzstan", - "LA" => "Lao People\"s Democratic Republic", + "LA" => "Lao People's Democratic Republic", "LV" => "Latvia", "LB" => "Lebanon", "LS" => "Lesotho", diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index e9cd97baf..e9a994346 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -214,10 +214,10 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:source", metrics, pagination) |> maybe_add_cr(site, query, pagination, "source", "visit:source") - |> transform_keys(%{"source" => "name", "visitors" => "count"}) + |> transform_keys(%{"source" => "name"}) if params["csv"] do - res |> to_csv(["name", "count", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) else json(conn, res) end @@ -237,10 +237,10 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium") - |> transform_keys(%{"utm_medium" => "name", "visitors" => "count"}) + |> transform_keys(%{"utm_medium" => "name"}) if params["csv"] do - res |> to_csv(["name", "count", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) else json(conn, res) end @@ -260,10 +260,10 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign") - |> transform_keys(%{"utm_campaign" => "name", "visitors" => "count"}) + |> transform_keys(%{"utm_campaign" => "name"}) if params["csv"] do - res |> to_csv(["name", "count", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) else json(conn, res) end @@ -283,10 +283,10 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source") - |> transform_keys(%{"utm_source" => "name", "visitors" => "count"}) + |> transform_keys(%{"utm_source" => "name"}) if params["csv"] do - res |> to_csv(["name", "count", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) else json(conn, res) end @@ -338,7 +338,7 @@ defmodule PlausibleWeb.Api.StatsController do referrers = Stats.breakdown(site, query, "visit:referrer", metrics, pagination) |> maybe_add_cr(site, query, pagination, "referrer", "visit:referrer") - |> transform_keys(%{"referrer" => "name", "visitors" => "count"}) + |> transform_keys(%{"referrer" => "name"}) %{"visitors" => %{"value" => total_visitors}} = Stats.aggregate(site, query, ["visitors"]) json(conn, %{referrers: referrers, total_visitors: total_visitors}) @@ -358,10 +358,10 @@ defmodule PlausibleWeb.Api.StatsController do pages = Stats.breakdown(site, query, "event:page", metrics, pagination) |> maybe_add_cr(site, query, pagination, "page", "event:page") - |> transform_keys(%{"page" => "name", "visitors" => "count"}) + |> transform_keys(%{"page" => "name"}) if params["csv"] do - pages |> to_csv(["name", "count", "bounce_rate", "time_on_page"]) + pages |> to_csv(["name", "visitors", "bounce_rate", "time_on_page"]) else json(conn, pages) end @@ -376,10 +376,14 @@ defmodule PlausibleWeb.Api.StatsController do entry_pages = Stats.breakdown(site, query, "visit:entry_page", metrics, pagination) |> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page") - |> transform_keys(%{"entry_page" => "name", "visits" => "entries", "visitors" => "count"}) + |> transform_keys(%{ + "entry_page" => "name", + "visitors" => "unique_entrances", + "visits" => "total_entrances" + }) if params["csv"] do - entry_pages |> to_csv(["name", "count", "entries", "visit_duration"]) + entry_pages |> to_csv(["name", "unique_entrances", "total_entrances", "visit_duration"]) else json(conn, entry_pages) end @@ -394,7 +398,11 @@ defmodule PlausibleWeb.Api.StatsController do exit_pages = Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page}) |> maybe_add_cr(site, query, {limit, page}, "exit_page", "visit:exit_page") - |> transform_keys(%{"exit_page" => "name", "visits" => "exits", "visitors" => "count"}) + |> transform_keys(%{ + "exit_page" => "name", + "visitors" => "unique_exits", + "visits" => "total_exits" + }) pages = Enum.map(exit_pages, & &1["name"]) @@ -413,7 +421,7 @@ defmodule PlausibleWeb.Api.StatsController do exit_rate = case Enum.find(total_pageviews, &(&1["page"] == exit_page["name"])) do %{"pageviews" => pageviews} -> - Float.floor(exit_page["exits"] / pageviews * 100) + Float.floor(exit_page["total_exits"] / pageviews * 100) nil -> nil @@ -423,7 +431,7 @@ defmodule PlausibleWeb.Api.StatsController do end) if params["csv"] do - exit_pages |> to_csv(["name", "count", "exits", "exit_rate"]) + exit_pages |> to_csv(["name", "unique_exits", "total_exits", "exit_rate"]) else json(conn, exit_pages) end @@ -437,16 +445,23 @@ defmodule PlausibleWeb.Api.StatsController do countries = Stats.breakdown(site, query, "visit:country", ["visitors"], pagination) |> maybe_add_cr(site, query, {300, 1}, "country", "visit:country") - |> transform_keys(%{"country" => "name", "visitors" => "count"}) - |> Enum.map(fn country -> - alpha3 = Stats.CountryName.to_alpha3(country["name"]) - Map.put(country, "name", alpha3) - end) + |> transform_keys(%{"country" => "name"}) |> add_percentages if params["csv"] do - countries |> to_csv(["name", "count"]) + countries + |> Enum.map(fn country -> + iso3166 = Stats.CountryName.from_iso3166(country["name"]) + Map.put(country, "name", iso3166) + end) + |> to_csv(["name", "visitors"]) else + countries = + Enum.map(countries, fn country -> + alpha3 = Stats.CountryName.to_alpha3(country["name"]) + Map.put(country, "name", alpha3) + end) + json(conn, countries) end end @@ -459,11 +474,11 @@ defmodule PlausibleWeb.Api.StatsController do browsers = Stats.breakdown(site, query, "visit:browser", ["visitors"], pagination) |> maybe_add_cr(site, query, pagination, "browser", "visit:browser") - |> transform_keys(%{"browser" => "name", "visitors" => "count"}) + |> transform_keys(%{"browser" => "name"}) |> add_percentages if params["csv"] do - browsers |> to_csv(["name", "count"]) + browsers |> to_csv(["name", "visitors"]) else json(conn, browsers) end @@ -477,7 +492,7 @@ defmodule PlausibleWeb.Api.StatsController do versions = Stats.breakdown(site, query, "visit:browser_version", ["visitors"], pagination) |> maybe_add_cr(site, query, pagination, "browser_version", "visit:browser_version") - |> transform_keys(%{"browser_version" => "name", "visitors" => "count"}) + |> transform_keys(%{"browser_version" => "name"}) |> add_percentages json(conn, versions) @@ -491,11 +506,11 @@ defmodule PlausibleWeb.Api.StatsController do systems = Stats.breakdown(site, query, "visit:os", ["visitors"], pagination) |> maybe_add_cr(site, query, pagination, "os", "visit:os") - |> transform_keys(%{"os" => "name", "visitors" => "count"}) + |> transform_keys(%{"os" => "name"}) |> add_percentages if params["csv"] do - systems |> to_csv(["name", "count"]) + systems |> to_csv(["name", "visitors"]) else json(conn, systems) end @@ -509,7 +524,7 @@ defmodule PlausibleWeb.Api.StatsController do versions = Stats.breakdown(site, query, "visit:os_version", ["visitors"], pagination) |> maybe_add_cr(site, query, pagination, "os_version", "visit:os_version") - |> transform_keys(%{"os_version" => "name", "visitors" => "count"}) + |> transform_keys(%{"os_version" => "name"}) |> add_percentages json(conn, versions) @@ -523,11 +538,11 @@ defmodule PlausibleWeb.Api.StatsController do sizes = Stats.breakdown(site, query, "visit:device", ["visitors"], pagination) |> maybe_add_cr(site, query, pagination, "device", "visit:device") - |> transform_keys(%{"device" => "name", "visitors" => "count"}) + |> transform_keys(%{"device" => "name"}) |> add_percentages if params["csv"] do - sizes |> to_csv(["name", "count"]) + sizes |> to_csv(["name", "visitors"]) else json(conn, sizes) end @@ -563,15 +578,19 @@ defmodule PlausibleWeb.Api.StatsController do conversions = Stats.breakdown(site, query, "event:goal", ["visitors", "events"], {100, 1}) - |> transform_keys(%{"goal" => "name", "visitors" => "count", "events" => "total_count"}) + |> transform_keys(%{ + "goal" => "name", + "count" => "unique_conversions", + "events" => "total_conversions" + }) |> Enum.map(fn goal -> goal |> Map.put(:prop_names, prop_names[goal["name"]]) - |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["count"])) + |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal["visitors"])) end) if params["csv"] do - conversions |> to_csv(["name", "count", "total_count"]) + conversions |> to_csv(["name", "unique_conversions", "total_conversions"]) else json(conn, conversions) end @@ -592,11 +611,10 @@ defmodule PlausibleWeb.Api.StatsController do Stats.breakdown(site, query, prop_name, ["visitors", "events"], pagination) |> transform_keys(%{ params["prop_name"] => "name", - "visitors" => "count", - "events" => "total_count" + "events" => "total_conversions" }) |> Enum.map(fn prop -> - Map.put(prop, "conversion_rate", calculate_cr(unique_visitors, prop["count"])) + Map.put(prop, "conversion_rate", calculate_cr(unique_visitors, prop["visitors"])) end) json(conn, props) @@ -636,10 +654,10 @@ defmodule PlausibleWeb.Api.StatsController do end defp add_percentages(stat_list) do - total = Enum.reduce(stat_list, 0, fn %{"count" => count}, total -> total + count end) + total = Enum.reduce(stat_list, 0, fn %{"visitors" => count}, total -> total + count end) Enum.map(stat_list, fn stat -> - Map.put(stat, "percentage", round(stat["count"] / total * 100)) + Map.put(stat, "percentage", round(stat["visitors"] / total * 100)) end) end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index ab5d71b41..42beaf8af 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -64,25 +64,30 @@ defmodule PlausibleWeb.StatsController do filename = "Plausible export #{params["domain"]} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.zip" - params = Map.put(params, "limit", "1000") - params = Map.put(params, "csv", "True") + params = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"}) csvs = [ - {'visitors.csv', visitors}, - {'sources.csv', Api.StatsController.sources(conn, params)}, - {'utm_mediums.csv', Api.StatsController.utm_mediums(conn, params)}, - {'utm_sources.csv', Api.StatsController.utm_sources(conn, params)}, - {'utm_campaigns.csv', Api.StatsController.utm_campaigns(conn, params)}, - {'pages.csv', Api.StatsController.pages(conn, params)}, - {'entry_pages.csv', Api.StatsController.entry_pages(conn, params)}, - {'exit_pages.csv', Api.StatsController.exit_pages(conn, params)}, - {'countries.csv', Api.StatsController.countries(conn, params)}, - {'browsers.csv', Api.StatsController.browsers(conn, params)}, - {'operating_systems.csv', Api.StatsController.operating_systems(conn, params)}, - {'devices.csv', Api.StatsController.screen_sizes(conn, params)}, - {'conversions.csv', Api.StatsController.conversions(conn, params)} + {'sources.csv', fn -> Api.StatsController.sources(conn, params) end}, + {'utm_mediums.csv', fn -> Api.StatsController.utm_mediums(conn, params) end}, + {'utm_sources.csv', fn -> Api.StatsController.utm_sources(conn, params) end}, + {'utm_campaigns.csv', fn -> Api.StatsController.utm_campaigns(conn, params) end}, + {'pages.csv', fn -> Api.StatsController.pages(conn, params) end}, + {'entry_pages.csv', fn -> Api.StatsController.entry_pages(conn, params) end}, + {'exit_pages.csv', fn -> Api.StatsController.exit_pages(conn, params) end}, + {'countries.csv', fn -> Api.StatsController.countries(conn, params) end}, + {'browsers.csv', fn -> Api.StatsController.browsers(conn, params) end}, + {'operating_systems.csv', fn -> Api.StatsController.operating_systems(conn, params) end}, + {'devices.csv', fn -> Api.StatsController.screen_sizes(conn, params) end}, + {'conversions.csv', fn -> Api.StatsController.conversions(conn, params) end} ] + csvs = + csvs + |> Enum.map(fn {file, task} -> {file, Task.async(task)} end) + |> Enum.map(fn {file, task} -> {file, Task.await(task)} end) + + csvs = [{'visitors.csv', visitors} | csvs] + {:ok, {_, zip_content}} = :zip.create(filename, csvs, [:memory]) conn diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 1aca02344..da4eccbbb 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -15,8 +15,8 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day") assert json_response(conn, 200) == [ - %{"name" => "Chrome", "count" => 2, "percentage" => 67}, - %{"name" => "Firefox", "count" => 1, "percentage" => 33} + %{"name" => "Chrome", "visitors" => 2, "percentage" => 67}, + %{"name" => "Firefox", "visitors" => 1, "percentage" => 33} ] end @@ -35,7 +35,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do %{ "name" => "Chrome", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "percentage" => 100, "conversion_rate" => 50.0 } @@ -63,8 +63,8 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do ) assert json_response(conn, 200) == [ - %{"name" => "78.0", "count" => 2, "percentage" => 67}, - %{"name" => "77.0", "count" => 1, "percentage" => 33} + %{"name" => "78.0", "visitors" => 2, "percentage" => 67}, + %{"name" => "77.0", "visitors" => 1, "percentage" => 33} ] end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index d0441bed2..a924067d0 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -35,15 +35,15 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert json_response(conn, 200) == [ %{ "name" => "Signup", - "count" => 2, - "total_count" => 3, + "visitors" => 2, + "total_conversions" => 3, "prop_names" => nil, "conversion_rate" => 50 }, %{ "name" => "Visit /register", - "count" => 2, - "total_count" => 2, + "visitors" => 2, + "total_conversions" => 2, "prop_names" => nil, "conversion_rate" => 50 } @@ -78,8 +78,8 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert json_response(conn, 200) == [ %{ "name" => "Signup", - "count" => 2, - "total_count" => 2, + "visitors" => 2, + "total_conversions" => 2, "prop_names" => ["variant"], "conversion_rate" => 50 } @@ -111,8 +111,18 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ) assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "B", "total_count" => 2, "conversion_rate" => 66.7}, - %{"count" => 1, "name" => "A", "total_count" => 1, "conversion_rate" => 33.3} + %{ + "visitors" => 2, + "name" => "B", + "total_conversions" => 2, + "conversion_rate" => 66.7 + }, + %{ + "visitors" => 1, + "name" => "A", + "total_conversions" => 1, + "conversion_rate" => 33.3 + } ] end @@ -137,8 +147,18 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ) assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "(none)", "total_count" => 2, "conversion_rate" => 66.7}, - %{"count" => 1, "name" => "A", "total_count" => 1, "conversion_rate" => 33.3} + %{ + "visitors" => 2, + "name" => "(none)", + "total_conversions" => 2, + "conversion_rate" => 66.7 + }, + %{ + "visitors" => 1, + "name" => "A", + "total_conversions" => 1, + "conversion_rate" => 33.3 + } ] end @@ -161,7 +181,12 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ) assert json_response(conn, 200) == [ - %{"count" => 1, "name" => "B", "total_count" => 1, "conversion_rate" => 50.0} + %{ + "visitors" => 1, + "name" => "B", + "total_conversions" => 1, + "conversion_rate" => 50.0 + } ] end end @@ -190,58 +215,58 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert json_response(conn, 200) == [ %{ "conversion_rate" => 100.0, - "count" => 8, + "visitors" => 8, "name" => "Visit /**", - "total_count" => 8, + "total_conversions" => 8, "prop_names" => nil }, %{ "conversion_rate" => 37.5, - "count" => 3, + "visitors" => 3, "name" => "Visit /*", - "total_count" => 3, + "total_conversions" => 3, "prop_names" => nil }, %{ "conversion_rate" => 37.5, - "count" => 3, + "visitors" => 3, "name" => "Visit /signup/**", - "total_count" => 3, + "total_conversions" => 3, "prop_names" => nil }, %{ "conversion_rate" => 25.0, - "count" => 2, + "visitors" => 2, "name" => "Visit /billing**/success", - "total_count" => 2, + "total_conversions" => 2, "prop_names" => nil }, %{ "conversion_rate" => 25.0, - "count" => 2, + "visitors" => 2, "name" => "Visit /reg*", - "total_count" => 2, + "total_conversions" => 2, "prop_names" => nil }, %{ "conversion_rate" => 12.5, - "count" => 1, + "visitors" => 1, "name" => "Visit /billing*/success", - "total_count" => 1, + "total_conversions" => 1, "prop_names" => nil }, %{ "conversion_rate" => 12.5, - "count" => 1, + "visitors" => 1, "name" => "Visit /register", - "total_count" => 1, + "total_conversions" => 1, "prop_names" => nil }, %{ "conversion_rate" => 12.5, - "count" => 1, + "visitors" => 1, "name" => "Visit /signup/*", - "total_count" => 1, + "total_conversions" => 1, "prop_names" => nil } ] diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index be345032d..9a63a7199 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -23,12 +23,12 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do assert json_response(conn, 200) == [ %{ "name" => "EST", - "count" => 2, + "visitors" => 2, "percentage" => 67 }, %{ "name" => "GBR", - "count" => 1, + "visitors" => 1, "percentage" => 33 } ] @@ -60,14 +60,14 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do %{ "name" => "GBR", "total_visitors" => 1, - "count" => 1, + "visitors" => 1, "percentage" => 50, "conversion_rate" => 100.0 }, %{ "name" => "EST", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "percentage" => 50, "conversion_rate" => 50.0 } diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index 8962b9e3e..4b9864dda 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -15,8 +15,8 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day") assert json_response(conn, 200) == [ - %{"name" => "Mac", "count" => 2, "percentage" => 67}, - %{"name" => "Android", "count" => 1, "percentage" => 33} + %{"name" => "Mac", "visitors" => 2, "percentage" => 67}, + %{"name" => "Android", "visitors" => 1, "percentage" => 33} ] end @@ -36,7 +36,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do %{ "name" => "Mac", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "percentage" => 100, "conversion_rate" => 50.0 } @@ -64,8 +64,8 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do ) assert json_response(conn, 200) == [ - %{"name" => "10.16", "count" => 2, "percentage" => 67}, - %{"name" => "10.15", "count" => 1, "percentage" => 33} + %{"name" => "10.16", "visitors" => 2, "percentage" => 67}, + %{"name" => "10.15", "visitors" => 1, "percentage" => 33} ] end end diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 468d1fbd1..e9db51ec9 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -19,9 +19,9 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do conn = get(conn, "/api/stats/#{site.domain}/pages?period=day") assert json_response(conn, 200) == [ - %{"count" => 3, "name" => "/"}, - %{"count" => 2, "name" => "/register"}, - %{"count" => 1, "name" => "/contact"} + %{"visitors" => 3, "name" => "/"}, + %{"visitors" => 2, "name" => "/register"}, + %{"visitors" => 1, "name" => "/contact"} ] end @@ -53,14 +53,14 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do %{ "bounce_rate" => 50.0, "time_on_page" => 900.0, - "count" => 2, + "visitors" => 2, "pageviews" => 2, "name" => "/" }, %{ "bounce_rate" => nil, "time_on_page" => nil, - "count" => 1, + "visitors" => 1, "pageviews" => 1, "name" => "/some-other-page" } @@ -77,8 +77,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do conn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime") assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "/page1"}, - %{"count" => 1, "name" => "/page2"} + %{"visitors" => 2, "name" => "/page1"}, + %{"visitors" => 1, "name" => "/page2"} ] end @@ -95,7 +95,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}") assert json_response(conn, 200) == [ - %{"total_visitors" => 3, "count" => 1, "name" => "/", "conversion_rate" => 33.3} + %{"total_visitors" => 3, "visitors" => 1, "name" => "/", "conversion_rate" => 33.3} ] end end @@ -137,14 +137,14 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do assert json_response(conn, 200) == [ %{ - "count" => 2, - "entries" => 2, + "unique_entrances" => 2, + "total_entrances" => 2, "name" => "/page1", "visit_duration" => 0 }, %{ - "count" => 1, - "entries" => 2, + "unique_entrances" => 1, + "total_entrances" => 2, "name" => "/page2", "visit_duration" => 450 } @@ -196,16 +196,16 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do assert json_response(conn, 200) == [ %{ "total_visitors" => 1, - "count" => 1, - "entries" => 1, + "unique_entrances" => 1, + "total_entrances" => 1, "name" => "/page2", "visit_duration" => 900, "conversion_rate" => 100.0 }, %{ "total_visitors" => 2, - "count" => 1, - "entries" => 1, + "unique_entrances" => 1, + "total_entrances" => 1, "name" => "/page1", "visit_duration" => 0, "conversion_rate" => 50.0 @@ -242,8 +242,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01") assert json_response(conn, 200) == [ - %{"name" => "/page1", "count" => 2, "exits" => 2, "exit_rate" => 66}, - %{"name" => "/page2", "count" => 1, "exits" => 1, "exit_rate" => 100} + %{"name" => "/page1", "unique_exits" => 2, "total_exits" => 2, "exit_rate" => 66}, + %{"name" => "/page2", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 100} ] end @@ -290,17 +290,17 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do assert json_response(conn, 200) == [ %{ "name" => "/exit1", - "count" => 1, + "unique_exits" => 1, "total_visitors" => 1, - "exits" => 1, + "total_exits" => 1, "exit_rate" => 50, "conversion_rate" => 100.0 }, %{ "name" => "/exit2", - "count" => 1, + "unique_exits" => 1, "total_visitors" => 1, - "exits" => 1, + "total_exits" => 1, "exit_rate" => 100, "conversion_rate" => 100.0 } @@ -345,8 +345,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ) assert json_response(conn, 200) == [ - %{"name" => "/exit1", "count" => 1, "exits" => 1, "exit_rate" => 50}, - %{"name" => "/exit2", "count" => 1, "exits" => 1, "exit_rate" => 100} + %{"name" => "/exit1", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 50}, + %{"name" => "/exit2", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 100} ] end end diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 22f15419e..e76d99da7 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -15,8 +15,8 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day") assert json_response(conn, 200) == [ - %{"name" => "Desktop", "count" => 2, "percentage" => 67}, - %{"name" => "Laptop", "count" => 1, "percentage" => 33} + %{"name" => "Desktop", "visitors" => 2, "percentage" => 67}, + %{"name" => "Laptop", "visitors" => 1, "percentage" => 33} ] end @@ -35,7 +35,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do %{ "name" => "Desktop", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "percentage" => 100, "conversion_rate" => 50.0 } diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 6b8b947d3..d16636730 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -25,8 +25,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources") assert json_response(conn, 200) == [ - %{"name" => "Google", "count" => 2}, - %{"name" => "DuckDuckGo", "count" => 1} + %{"name" => "Google", "visitors" => 2}, + %{"name" => "DuckDuckGo", "visitors" => 1} ] end @@ -60,13 +60,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == [ %{ "name" => "Google", - "count" => 1, + "visitors" => 1, "bounce_rate" => 0, "visit_duration" => 900 }, %{ "name" => "DuckDuckGo", - "count" => 1, + "visitors" => 1, "bounce_rate" => 100, "visit_duration" => 0 } @@ -95,8 +95,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources?period=realtime") assert json_response(conn, 200) == [ - %{"name" => "Google", "count" => 2}, - %{"name" => "DuckDuckGo", "count" => 1} + %{"name" => "Google", "visitors" => 2}, + %{"name" => "DuckDuckGo", "visitors" => 1} ] end @@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2") assert json_response(conn, 200) == [ - %{"name" => "DuckDuckGo", "count" => 1} + %{"name" => "DuckDuckGo", "visitors" => 1} ] end @@ -135,7 +135,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do conn = get(conn, "/api/stats/#{site.domain}/sources?filters=#{filters}") assert json_response(conn, 200) == [ - %{"name" => "Google", "count" => 1} + %{"name" => "Google", "visitors" => 1} ] end end @@ -170,13 +170,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == [ %{ "name" => "social", - "count" => 1, + "visitors" => 1, "bounce_rate" => 0, "visit_duration" => 900 }, %{ "name" => "email", - "count" => 1, + "visitors" => 1, "bounce_rate" => 100, "visit_duration" => 0 } @@ -218,13 +218,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == [ %{ "name" => "august", - "count" => 2, + "visitors" => 2, "bounce_rate" => 100, "visit_duration" => 0 }, %{ "name" => "profile", - "count" => 1, + "visitors" => 1, "bounce_rate" => 0, "visit_duration" => 900 } @@ -266,13 +266,13 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == [ %{ "name" => "newsletter", - "count" => 2, + "visitors" => 2, "bounce_rate" => 100, "visit_duration" => 0 }, %{ "name" => "Twitter", - "count" => 1, + "visitors" => 1, "bounce_rate" => 0, "visit_duration" => 900 } @@ -313,7 +313,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do %{ "name" => "Twitter", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "conversion_rate" => 50.0 } ] @@ -349,7 +349,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do %{ "name" => "Twitter", "total_visitors" => 2, - "count" => 1, + "visitors" => 1, "conversion_rate" => 50.0 } ] @@ -388,8 +388,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == %{ "total_visitors" => 3, "referrers" => [ - %{"name" => "10words.com", "count" => 2}, - %{"name" => "10words.com/page1", "count" => 1} + %{"name" => "10words.com", "visitors" => 2}, + %{"name" => "10words.com/page1", "visitors" => 1} ] } end @@ -431,7 +431,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "referrers" => [ %{ "name" => "10words.com", - "count" => 2, + "visitors" => 2, "bounce_rate" => 50.0, "visit_duration" => 450 } @@ -501,7 +501,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "name" => "10words.com", "total_visitors" => 2, "conversion_rate" => 50.0, - "count" => 1 + "visitors" => 1 } ] } @@ -542,7 +542,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "name" => "10words.com", "total_visitors" => 2, "conversion_rate" => 50.0, - "count" => 1 + "visitors" => 1 } ] } diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 67aca7001..ff0d68e03 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -61,37 +61,7 @@ defmodule PlausibleWeb.StatsControllerTest do ]) conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20") - assert conn.status == 200 - - assert {"content-type", "application/zip; charset=utf-8"} = - List.keyfind(conn.resp_headers, "content-type", 0) - - {:ok, zip} = :zip.unzip(response(conn, 200), [:memory]) - - assert_csv( - zip, - 'visitors.csv', - "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n" - ) - - assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n") - assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n") - assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n") - - assert_csv( - zip, - 'utm_campaigns.csv', - "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n" - ) - - assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n") - assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n") - assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n") - assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n") - assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n") + assert_default_zip(conn) end end @@ -113,55 +83,75 @@ defmodule PlausibleWeb.StatsControllerTest do ]) conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20") - assert conn.status == 200 - - assert {"content-type", "application/zip; charset=utf-8"} = - List.keyfind(conn.resp_headers, "content-type", 0) - - {:ok, zip} = :zip.unzip(response(conn, 200), [:memory]) - - assert_csv( - zip, - 'visitors.csv', - "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n" - ) - - assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n") - assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n") - assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n") - - assert_csv( - zip, - 'utm_campaigns.csv', - "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n" - ) - - assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,2,,\r\n") - assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,2,2,0\r\n") - assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,2,2,100.0\r\n") - assert_csv(zip, 'countries.csv', "name,count\r\nEST,1\r\n") - assert_csv(zip, 'browsers.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'operating_systems.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'devices.csv', "name,count\r\n,2\r\n") - assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n") + assert_default_zip(conn) end end + defp assert_default_zip(conn) do + assert conn.status == 200 + + assert {"content-type", "application/zip; charset=utf-8"} = + List.keyfind(conn.resp_headers, "content-type", 0) + + {:ok, zip} = :zip.unzip(response(conn, 200), [:memory]) + + assert_csv( + zip, + 'visitors.csv', + "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n" + ) + + assert_csv( + zip, + 'sources.csv', + "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n" + ) + + assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n") + assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n") + + assert_csv( + zip, + 'utm_campaigns.csv', + "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n" + ) + + assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,2,100,\r\n") + + assert_csv( + zip, + 'entry_pages.csv', + "name,unique_entrances,total_entrances,visit_duration\r\n/,2,2,0\r\n" + ) + + assert_csv( + zip, + 'exit_pages.csv', + "name,unique_exits,total_exits,exit_rate\r\n/,2,2,100.0\r\n" + ) + + assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,1\r\n") + assert_csv(zip, 'browsers.csv', "name,visitors\r\n,2\r\n") + assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,2\r\n") + assert_csv(zip, 'devices.csv', "name,visitors\r\n,2\r\n") + assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n") + end + describe "GET /:website/export - for past 6 months" do setup [:create_user, :create_new_site, :log_in] test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - timestamp: relative_time(minutes: -1) + timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1) ), build(:pageview, - timestamp: relative_time(months: -1), + timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1), country_code: "EE", browser: "ABrowserName" ), build(:pageview, - timestamp: relative_time(months: -5), + timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5), utm_campaign: "ads", country_code: "EE", referrer_source: "Google", @@ -183,24 +173,40 @@ defmodule PlausibleWeb.StatsControllerTest do "date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n" ) - assert_csv(zip, 'sources.csv', "name,count,bounce_rate,visit_duration\r\nGoogle,1,,\r\n") - assert_csv(zip, 'utm_mediums.csv', "name,count,bounce_rate,visit_duration\r\n") - assert_csv(zip, 'utm_sources.csv', "name,count,bounce_rate,visit_duration\r\n") + assert_csv( + zip, + 'sources.csv', + "name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n" + ) + + assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n") + assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n") assert_csv( zip, 'utm_campaigns.csv', - "name,count,bounce_rate,visit_duration\r\nads,1,100,0\r\n" + "name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n" + ) + + assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,3,100,\r\n") + + assert_csv( + zip, + 'entry_pages.csv', + "name,unique_entrances,total_entrances,visit_duration\r\n/,3,3,0\r\n" + ) + + assert_csv( + zip, + 'exit_pages.csv', + "name,unique_exits,total_exits,exit_rate\r\n/,3,3,100.0\r\n" ) - assert_csv(zip, 'pages.csv', "name,count,bounce_rate,time_on_page\r\n/,3,,\r\n") - assert_csv(zip, 'entry_pages.csv', "name,count,entries,visit_duration\r\n/,3,3,0\r\n") - assert_csv(zip, 'exit_pages.csv', "name,count,exits,exit_rate\r\n/,3,3,100.0\r\n") - assert_csv(zip, 'countries.csv', "name,count\r\nEST,2\r\n") - assert_csv(zip, 'browsers.csv', "name,count\r\nABrowserName,2\r\n,1\r\n") - assert_csv(zip, 'operating_systems.csv', "name,count\r\n,3\r\n") - assert_csv(zip, 'devices.csv', "name,count\r\n,3\r\n") - assert_csv(zip, 'conversions.csv', "name,count,total_count\r\n") + assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,2\r\n") + assert_csv(zip, 'browsers.csv', "name,visitors\r\nABrowserName,2\r\n,1\r\n") + assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,3\r\n") + assert_csv(zip, 'devices.csv', "name,visitors\r\n,3\r\n") + assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n") end end