From 49f153c38075db0a3f8431c17dfc3858b2da8e6c Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 10 Sep 2020 10:42:43 +0300 Subject: [PATCH] Paginate referrers (#313) * Add pagination for referrers * Fix top referrers for goal --- assets/js/dashboard/stats/modals/referrers.js | 82 +++++++++++-------- lib/plausible/stats/clickhouse.ex | 14 ++-- .../controllers/api/stats_controller.ex | 7 +- lib/workers/send_email_report.ex | 2 +- .../api/stats_controller/referrers_test.exs | 9 ++ 5 files changed, 73 insertions(+), 41 deletions(-) diff --git a/assets/js/dashboard/stats/modals/referrers.js b/assets/js/dashboard/stats/modals/referrers.js index ad4136304..97213ad45 100644 --- a/assets/js/dashboard/stats/modals/referrers.js +++ b/assets/js/dashboard/stats/modals/referrers.js @@ -12,26 +12,39 @@ class ReferrersModal extends React.Component { super(props) this.state = { loading: true, - query: parseQuery(props.location.search, props.site) + referrers: [], + query: parseQuery(props.location.search, props.site), + page: 1, + moreResultsAvailable: false } } - componentDidMount() { - if (this.state.query.filters.goal) { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers`, this.state.query, {limit: 100}) - .then((res) => this.setState({loading: false, referrers: res})) + loadReferrers() { + const {query, page, referrers} = this.state + + if (query.filters.goal) { + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers`, query, {limit: 100, page: page}) + .then((res) => this.setState({loading: false, referrers: referrers.concat(res), moreResultsAvailable: res.length === 100})) } else { const include = this.showExtra() ? 'bounce_rate,visit_duration' : null - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers`, this.state.query, { limit: 100, include: include, show_noref: true}) - .then((res) => this.setState({loading: false, referrers: res})) + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers`, query, {limit: 100, page: page, include: include, show_noref: true}) + .then((res) => this.setState({loading: false, referrers: referrers.concat(res), moreResultsAvailable: res.length === 100})) } } + componentDidMount() { + this.loadReferrers() + } + showExtra() { return this.state.query.period !== 'realtime' && !this.state.query.filters.goal } + loadMore() { + this.setState({loading: true, page: this.state.page + 1}, this.loadReferrers.bind(this)) + } + formatBounceRate(page) { if (typeof(page.bounce_rate) === 'number') { return page.bounce_rate + '%' @@ -69,33 +82,16 @@ class ReferrersModal extends React.Component { return this.state.query.period === 'realtime' ? 'Current visitors' : 'Visitors' } - renderBody() { + renderLoading() { if (this.state.loading) { + return
+ } else if (this.state.moreResultsAvailable) { return ( -
- ) - } else if (this.state.referrers) { - return ( - -

Top Sources

- -
-
- - - - - - {this.showExtra() && } - {this.showExtra() && } - - - - { this.state.referrers.map(this.renderReferrer.bind(this)) } - -
Referrer{this.label()}Bounce rateVisit duration
-
-
+
+ +
) } } @@ -103,7 +99,27 @@ class ReferrersModal extends React.Component { render() { return ( - { this.renderBody() } +

Top Sources

+ +
+ +
+ + + + + + {this.showExtra() && } + {this.showExtra() && } + + + + { this.state.referrers.map(this.renderReferrer.bind(this)) } + +
Referrer{this.label()}Bounce rateVisit duration
+
+ + { this.renderLoading() }
) } diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index aaceef857..34c4f1469 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -196,13 +196,14 @@ defmodule Plausible.Stats.Clickhouse do res["visitors"] end - def top_referrers_for_goal(site, query, limit \\ 5) do + def top_referrers_for_goal(site, query, limit, page) do converted_sessions = from( from e in base_query(site, query), select: %{session_id: e.session_id} ) + offset = (page - 1) * limit Plausible.Clickhouse.all( from s in Plausible.ClickhouseSession, join: cs in subquery(converted_sessions), @@ -213,19 +214,22 @@ defmodule Plausible.Stats.Clickhouse do where: s.referrer_source != "", group_by: s.referrer_source, order_by: [desc: fragment("count")], - limit: ^limit + limit: ^limit, + offset: ^offset ) |> Enum.map(fn ref -> Map.update(ref, "url", nil, fn url -> url && URI.parse("http://" <> url).host end) end) end - def top_referrers(site, query, limit \\ 5, show_noref \\ false, include \\ []) do + def top_referrers(site, query, limit, page, show_noref \\ false, include \\ []) do + offset = (page - 1) * limit referrers = from(s in base_session_query(site, query), group_by: s.referrer_source, - order_by: [desc: fragment("count")], - limit: ^limit + order_by: [desc: fragment("count"), asc: fragment("min(start)")], + limit: ^limit, + offset: ^offset ) referrers = if show_noref do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index ccefef313..8750f5636 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -130,15 +130,18 @@ defmodule PlausibleWeb.Api.StatsController do query = Query.from(site.timezone, params) include = if params["include"], do: String.split(params["include"], ","), else: [] limit = if params["limit"], do: String.to_integer(params["limit"]) + page = if params["page"], do: String.to_integer(params["page"]) show_noref = params["show_noref"] == "true" - json(conn, Stats.top_referrers(site, query, limit || 9, show_noref, include)) + json(conn, Stats.top_referrers(site, query, limit || 9, page || 1, show_noref, include)) end def referrers_for_goal(conn, params) do site = conn.assigns[:site] query = Query.from(site.timezone, params) + limit = if params["limit"], do: String.to_integer(params["limit"]) + page = if params["page"], do: String.to_integer(params["page"]) - json(conn, Stats.top_referrers_for_goal(site, query, params["limit"] || 9)) + json(conn, Stats.top_referrers_for_goal(site, query, limit || 9, page || 1)) end @google_api Application.fetch_env!(:plausible, :google_api) diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 2d4d7771f..47df0a25e 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -55,7 +55,7 @@ defmodule Plausible.Workers.SendEmailReport do bounce_rate = Stats.bounce_rate(site, query) prev_bounce_rate = Stats.bounce_rate(site, Query.shift_back(query)) change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate - referrers = Stats.top_referrers(site, query, 5, []) + referrers = Stats.top_referrers(site, query, 5, 1, []) pages = Stats.top_pages(site, query, 5, []) user = Plausible.Auth.find_user_by(email: email) login_link = user && Plausible.Sites.is_owner?(user.id, site) diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index f40d3968d..42a8cb1a7 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -47,8 +47,17 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do %{"name" => "Bing", "count" => 1, "url" => "bing.com"} ] end + + test "can paginate the results", %{conn: conn, site: site} do + conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&limit=1&page=2") + + assert json_response(conn, 200) == [ + %{"name" => "Bing", "count" => 1, "url" => ""} + ] + end end + describe "GET /api/stats/:domain/goal/referrers" do setup [:create_user, :log_in, :create_site]