More improvements for CSV export (#1446)

* Round time_on_page metric returned by 'pages' API

This rounds the `time_on_page` metric returned as part of the `pages`
API to the nearest second. While this shows no apparent change in the
web UI, it removes the decimal from the exported data in `pages.csv`.

* Tidy up export tests

* Round time_on_page in db query

* Add goal to CSV export tests

* Display CSV export spinner until download is ready

The mechanism used to make the page aware that the download is ready is:

- Client code sets a cookie and requests download.
- Server code handles download and removes cookie when complete.
- Client code polls every 1s to check the cookies, removing spinner when
  the export cookie is removed.
pull/1456/head
Matt Colligan 5 years ago committed by GitHub
parent 5c02811d26
commit 29e37c8342

@ -304,12 +304,18 @@ class LineGraph extends React.Component {
return stats
}
pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(this.pollExportReady.bind(this), 1000);
} else {
this.setState({exported: false})
}
}
downloadSpinner() {
this.setState({exported: true});
setTimeout(
() => this.setState({exported: false}),
4000
);
document.cookie = "exporting=";
setTimeout(this.pollExportReady.bind(this), 1000);
}
downloadLink() {

@ -230,7 +230,7 @@ defmodule Plausible.Stats.Breakdown do
time_query = "
SELECT
p,
sum(td)/count(case when p2 != p then 1 end) as avgTime
round(sum(td)/count(case when p2 != p then 1 end)) as avgTime
FROM
(SELECT
p,

@ -93,6 +93,7 @@ defmodule PlausibleWeb.StatsController do
conn
|> put_resp_content_type("application/zip")
|> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"")
|> delete_resp_cookie("exporting")
|> send_resp(200, zip_content)
end

@ -0,0 +1,3 @@
name,visitors
ABrowserName,2
,1
1 name visitors
2 ABrowserName 2
3 1

@ -0,0 +1,2 @@
name,unique_conversions,total_conversions
Visit /some-other-page,1,1
1 name unique_conversions total_conversions
2 Visit /some-other-page 1 1

@ -0,0 +1,2 @@
name,visitors
Estonia,2
1 name visitors
2 Estonia 2

@ -0,0 +1,2 @@
name,visitors
,3
1 name visitors
2 3

@ -0,0 +1,2 @@
name,unique_entrances,total_entrances,visit_duration
/,3,3,20
1 name unique_entrances total_entrances visit_duration
2 / 3 3 20

@ -0,0 +1,3 @@
name,unique_exits,total_exits,exit_rate
/,2,2,66.0
/some-other-page,1,1,100.0
1 name unique_exits total_exits exit_rate
2 / 2 2 66.0
3 /some-other-page 1 1 100.0

@ -0,0 +1,2 @@
name,visitors
,3
1 name visitors
2 3

@ -0,0 +1,3 @@
name,visitors,bounce_rate,time_on_page
/,3,67,
/some-other-page,1,,60.0
1 name visitors bounce_rate time_on_page
2 / 3 67
3 /some-other-page 1 60.0

@ -0,0 +1,2 @@
name,visitors,bounce_rate,visit_duration
Google,1,0,60
1 name visitors bounce_rate visit_duration
2 Google 1 0 60

@ -0,0 +1,2 @@
name,visitors,bounce_rate,visit_duration
ads,1,100,0
1 name visitors bounce_rate visit_duration
2 ads 1 100 0

@ -0,0 +1 @@
name,visitors,bounce_rate,visit_duration
1 name visitors bounce_rate visit_duration

@ -0,0 +1 @@
name,visitors,bounce_rate,visit_duration
1 name visitors bounce_rate visit_duration

@ -0,0 +1,32 @@
date,visitors,pageviews,bounce_rate,visit_duration
2021-09-20,1,1,100,0
2021-09-21,0,0,,
2021-09-22,0,0,,
2021-09-23,0,0,,
2021-09-24,0,0,,
2021-09-25,0,0,,
2021-09-26,0,0,,
2021-09-27,0,0,,
2021-09-28,0,0,,
2021-09-29,0,0,,
2021-09-30,0,0,,
2021-10-01,0,0,,
2021-10-02,0,0,,
2021-10-03,0,0,,
2021-10-04,0,0,,
2021-10-05,0,0,,
2021-10-06,0,0,,
2021-10-07,0,0,,
2021-10-08,0,0,,
2021-10-09,0,0,,
2021-10-10,0,0,,
2021-10-11,0,0,,
2021-10-12,0,0,,
2021-10-13,0,0,,
2021-10-14,0,0,,
2021-10-15,0,0,,
2021-10-16,0,0,,
2021-10-17,0,0,,
2021-10-18,0,0,,
2021-10-19,1,1,100,0
2021-10-20,1,2,0,60
1 date visitors pageviews bounce_rate visit_duration
2 2021-09-20 1 1 100 0
3 2021-09-21 0 0
4 2021-09-22 0 0
5 2021-09-23 0 0
6 2021-09-24 0 0
7 2021-09-25 0 0
8 2021-09-26 0 0
9 2021-09-27 0 0
10 2021-09-28 0 0
11 2021-09-29 0 0
12 2021-09-30 0 0
13 2021-10-01 0 0
14 2021-10-02 0 0
15 2021-10-03 0 0
16 2021-10-04 0 0
17 2021-10-05 0 0
18 2021-10-06 0 0
19 2021-10-07 0 0
20 2021-10-08 0 0
21 2021-10-09 0 0
22 2021-10-10 0 0
23 2021-10-11 0 0
24 2021-10-12 0 0
25 2021-10-13 0 0
26 2021-10-14 0 0
27 2021-10-15 0 0
28 2021-10-16 0 0
29 2021-10-17 0 0
30 2021-10-18 0 0
31 2021-10-19 1 1 100 0
32 2021-10-20 1 2 0 60

@ -0,0 +1,3 @@
name,visitors
ABrowserName,3
,1
1 name visitors
2 ABrowserName 3
3 1

@ -0,0 +1,2 @@
name,unique_conversions,total_conversions
Visit /some-other-page,1,1
1 name unique_conversions total_conversions
2 Visit /some-other-page 1 1

@ -0,0 +1,2 @@
name,visitors
Estonia,3
1 name visitors
2 Estonia 3

@ -0,0 +1,2 @@
name,visitors
,4
1 name visitors
2 4

@ -0,0 +1,2 @@
name,unique_entrances,total_entrances,visit_duration
/,4,4,15
1 name unique_entrances total_entrances visit_duration
2 / 4 4 15

@ -0,0 +1,3 @@
name,unique_exits,total_exits,exit_rate
/,3,3,75.0
/some-other-page,1,1,100.0
1 name unique_exits total_exits exit_rate
2 / 3 3 75.0
3 /some-other-page 1 1 100.0

@ -0,0 +1,2 @@
name,visitors
,4
1 name visitors
2 4

@ -0,0 +1,3 @@
name,visitors,bounce_rate,time_on_page
/,4,75,
/some-other-page,1,,60.0
1 name visitors bounce_rate time_on_page
2 / 4 75
3 /some-other-page 1 60.0

@ -0,0 +1,2 @@
name,visitors,bounce_rate,visit_duration
Google,2,50,30
1 name visitors bounce_rate visit_duration
2 Google 2 50 30

@ -0,0 +1,2 @@
name,visitors,bounce_rate,visit_duration
ads,2,100,0
1 name visitors bounce_rate visit_duration
2 ads 2 100 0

@ -0,0 +1 @@
name,visitors,bounce_rate,visit_duration
1 name visitors bounce_rate visit_duration

@ -0,0 +1 @@
name,visitors,bounce_rate,visit_duration
1 name visitors bounce_rate visit_duration

@ -0,0 +1,7 @@
date,visitors,pageviews,bounce_rate,visit_duration
2021-05-01,1,1,100,0
2021-06-01,0,0,,
2021-07-01,0,0,,
2021-08-01,0,0,,
2021-09-01,1,1,100,0
2021-10-01,2,3,50,30
1 date visitors pageviews bounce_rate visit_duration
2 2021-05-01 1 1 100 0
3 2021-06-01 0 0
4 2021-07-01 0 0
5 2021-08-01 0 0
6 2021-09-01 1 1 100 0
7 2021-10-01 2 3 50 30

@ -48,20 +48,9 @@ defmodule PlausibleWeb.StatsControllerTest do
setup [:create_user, :create_new_site, :log_in]
test "exports data in zipped csvs", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
country_code: "EE",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
referrer_source: "Google"
),
build(:pageview,
utm_campaign: "ads",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
)
])
populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20")
assert_default_zip(conn)
assert_zip(conn, "30d")
end
end
@ -70,149 +59,79 @@ defmodule PlausibleWeb.StatsControllerTest do
site = insert(:site, domain: "new-site.com")
link = insert(:shared_link, site: site)
populate_stats(site, [
build(:pageview,
country_code: "EE",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
referrer_source: "Google"
),
build(:pageview,
utm_campaign: "ads",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1)
)
])
populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?auth=#{link.slug}&date=2021-10-20")
assert_default_zip(conn)
assert_zip(conn, "30d")
end
end
defp assert_default_zip(conn) do
assert conn.status == 200
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
assert_csv(
zip,
'visitors.csv',
"date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-09-20,0,0,,\r\n2021-09-21,0,0,,\r\n2021-09-22,0,0,,\r\n2021-09-23,0,0,,\r\n2021-09-24,0,0,,\r\n2021-09-25,0,0,,\r\n2021-09-26,0,0,,\r\n2021-09-27,0,0,,\r\n2021-09-28,0,0,,\r\n2021-09-29,0,0,,\r\n2021-09-30,0,0,,\r\n2021-10-01,0,0,,\r\n2021-10-02,0,0,,\r\n2021-10-03,0,0,,\r\n2021-10-04,0,0,,\r\n2021-10-05,0,0,,\r\n2021-10-06,0,0,,\r\n2021-10-07,0,0,,\r\n2021-10-08,0,0,,\r\n2021-10-09,0,0,,\r\n2021-10-10,0,0,,\r\n2021-10-11,0,0,,\r\n2021-10-12,0,0,,\r\n2021-10-13,0,0,,\r\n2021-10-14,0,0,,\r\n2021-10-15,0,0,,\r\n2021-10-16,0,0,,\r\n2021-10-17,0,0,,\r\n2021-10-18,0,0,,\r\n2021-10-19,1,1,100,0\r\n2021-10-20,1,1,100,0\r\n"
)
assert_csv(
zip,
'sources.csv',
"name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"
)
assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")
assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")
assert_csv(
zip,
'utm_campaigns.csv',
"name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
)
assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,2,100,\r\n")
assert_csv(
zip,
'entry_pages.csv',
"name,unique_entrances,total_entrances,visit_duration\r\n/,2,2,0\r\n"
)
assert_csv(
zip,
'exit_pages.csv',
"name,unique_exits,total_exits,exit_rate\r\n/,2,2,100.0\r\n"
)
assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,1\r\n")
assert_csv(zip, 'browsers.csv', "name,visitors\r\n,2\r\n")
assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,2\r\n")
assert_csv(zip, 'devices.csv', "name,visitors\r\n,2\r\n")
assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")
end
describe "GET /:website/export - for past 6 months" do
setup [:create_user, :create_new_site, :log_in]
test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1)
),
build(:pageview,
timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),
country_code: "EE",
browser: "ABrowserName"
),
build(:pageview,
timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),
utm_campaign: "ads",
country_code: "EE",
referrer_source: "Google",
browser: "ABrowserName"
)
])
populate_exported_stats(site)
conn = get(conn, "/" <> site.domain <> "/export?period=6mo&date=2021-10-20")
assert conn.status == 200
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
assert_zip(conn, "6m")
end
end
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
defp assert_zip(conn, folder) do
assert conn.status == 200
assert_csv(
zip,
'visitors.csv',
"date,visitors,pageviews,bounce_rate,visit_duration\r\n2021-05-01,1,1,100,0\r\n2021-06-01,0,0,,\r\n2021-07-01,0,0,,\r\n2021-08-01,0,0,,\r\n2021-09-01,1,1,100,0\r\n2021-10-01,1,1,100,0\r\n"
)
assert {"content-type", "application/zip; charset=utf-8"} =
List.keyfind(conn.resp_headers, "content-type", 0)
assert_csv(
zip,
'sources.csv',
"name,visitors,bounce_rate,visit_duration\r\nGoogle,1,100,0\r\n"
)
{:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
assert_csv(zip, 'utm_mediums.csv', "name,visitors,bounce_rate,visit_duration\r\n")
assert_csv(zip, 'utm_sources.csv', "name,visitors,bounce_rate,visit_duration\r\n")
folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")
assert_csv(
zip,
'utm_campaigns.csv',
"name,visitors,bounce_rate,visit_duration\r\nads,1,100,0\r\n"
)
Enum.map(zip, &assert_csv(&1, folder))
end
assert_csv(zip, 'pages.csv', "name,visitors,bounce_rate,time_on_page\r\n/,3,100,\r\n")
defp assert_csv({file, downloaded}, folder) do
file = Path.expand(file, folder)
assert_csv(
zip,
'entry_pages.csv',
"name,unique_entrances,total_entrances,visit_duration\r\n/,3,3,0\r\n"
)
{:ok, content} = File.read(file)
assert downloaded == content
end
assert_csv(
zip,
'exit_pages.csv',
"name,unique_exits,total_exits,exit_rate\r\n/,3,3,100.0\r\n"
defp populate_exported_stats(site) do
populate_stats(site, [
build(:pageview,
country_code: "EE",
pathname: "/",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1),
referrer_source: "Google",
user_id: 123
),
build(:pageview,
country_code: "EE",
pathname: "/some-other-page",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -2),
referrer_source: "Google",
user_id: 123
),
build(:pageview,
pathname: "/",
utm_campaign: "ads",
timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1),
browser: "ABrowserName"
),
build(:pageview,
timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1),
country_code: "EE",
browser: "ABrowserName"
),
build(:pageview,
timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5),
utm_campaign: "ads",
country_code: "EE",
referrer_source: "Google",
browser: "ABrowserName"
)
])
assert_csv(zip, 'countries.csv', "name,visitors\r\nEstonia,2\r\n")
assert_csv(zip, 'browsers.csv', "name,visitors\r\nABrowserName,2\r\n,1\r\n")
assert_csv(zip, 'operating_systems.csv', "name,visitors\r\n,3\r\n")
assert_csv(zip, 'devices.csv', "name,visitors\r\n,3\r\n")
assert_csv(zip, 'conversions.csv', "name,unique_conversions,total_conversions\r\n")
end
end
defp assert_csv(zip, fileName, string) do
{_, contents} = List.keyfind(zip, fileName, 0)
assert to_string(contents) == string
insert(:goal, %{domain: site.domain, page_path: "/some-other-page"})
end
describe "GET /share/:slug" do

Loading…
Cancel
Save