From 99bc20a8b37dc22b02197c5ed569c81ac04114db Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 30 Jul 2020 11:18:28 +0300 Subject: [PATCH] Referrer drilldown (#252) * Referrer drilldown * Render external links for referrer drilldown modal * Rename top pages to entry pages * Add google keywords to referrer drilldown * Fix link in source modal * Remove back link from referrer drilldown * Fix tests * Mix format * Remove unused function * Get top stats from the events table * Revert "Get top stats from the events table" This reverts commit 67384e2d3580651565e80c11fc05bf48a6af0803. * Fix pages modal title * Improve pages report in referrer drilldown * Fix label in referrer drilldown --- assets/js/dashboard/filters.js | 36 ++- assets/js/dashboard/query.js | 6 +- assets/js/dashboard/stats/conversions.js | 2 +- assets/js/dashboard/stats/modals/pages.js | 26 +- .../stats/modals/referrer-drilldown.js | 26 +- assets/js/dashboard/stats/modals/referrers.js | 7 +- assets/js/dashboard/stats/pages.js | 19 +- assets/js/dashboard/stats/referrers.js | 69 +++++- assets/js/dashboard/stats/visitor-graph.js | 5 +- lib/plausible/billing/billing.ex | 3 +- lib/plausible/google/api.ex | 4 +- lib/plausible/stats/clickhouse.ex | 233 ++++++++++++------ lib/plausible/stats/query.ex | 5 +- .../controllers/api/stats_controller.ex | 33 +-- lib/plausible_web/router.ex | 1 + test/plausible/stats/query_test.exs | 7 + .../api/stats_controller/main_graph_test.exs | 4 +- .../api/stats_controller/pages_test.exs | 89 ++++--- .../api/stats_controller/referrers_test.exs | 7 +- test/support/clickhouse_setup.ex | 27 ++ test/support/google_api_mock.ex | 2 +- 21 files changed, 433 insertions(+), 178 deletions(-) diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 46d617b61..137e5844c 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -2,17 +2,39 @@ import React from 'react'; import { withRouter } from 'react-router-dom' import {removeQueryParam} from './query' +function filterText(key, value) { + if (key === "goal") { + return Completed goal {value} + } + if (key === "source") { + return Source: {value} + } + if (key === "referrer") { + return Referrer: {value} + } +} + +function renderFilter(history, [key, value]) { + function removeFilter() { + history.push({search: removeQueryParam(location.search, key)}) + } + + return ( + + {filterText(key, value)} ✕ + + ) +} + function Filters({query, history, location}) { - if (query.filters.goal) { - function removeGoal() { - history.push({search: removeQueryParam(location.search, 'goal')}) - } + const appliedFilters = Object.keys(query.filters) + .map((key) => [key, query.filters[key]]) + .filter(([key, value]) => !!value) + if (appliedFilters.length > 0) { return (
- - Completed goal {query.filters.goal} ✕ - + { appliedFilters.map((filter) => renderFilter(history, filter)) }
) } diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index 070c9edbf..4f7274958 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -22,7 +22,11 @@ export function parseQuery(querystring, site) { date: q.get('date') ? parseUTCDate(q.get('date')) : nowInOffset(site.offset), from: q.get('from') ? parseUTCDate(q.get('from')) : undefined, to: q.get('to') ? parseUTCDate(q.get('to')) : undefined, - filters: {'goal': q.get('goal')} + filters: { + 'goal': q.get('goal'), + 'source': q.get('source'), + 'referrer': q.get('referrer') + } } } diff --git a/assets/js/dashboard/stats/conversions.js b/assets/js/dashboard/stats/conversions.js index f87f859c1..789deb38b 100644 --- a/assets/js/dashboard/stats/conversions.js +++ b/assets/js/dashboard/stats/conversions.js @@ -29,7 +29,7 @@ export default class Conversions extends React.Component { } renderGoalText(goalName) { - if (this.props.query.period === 'realtime') { + if (this.props.query.period === 'realtime' || this.props.query.filters['source'] || this.props.query.filters['referrer']) { return {goalName} } else { const query = new URLSearchParams(window.location.search) diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js index e651e3ae2..ec7c7d89d 100644 --- a/assets/js/dashboard/stats/modals/pages.js +++ b/assets/js/dashboard/stats/modals/pages.js @@ -16,16 +16,21 @@ class PagesModal extends React.Component { } componentDidMount() { - const include = this.showExtra() ? 'bounce_rate,unique_visitors' : null + const include = this.showBounceRate() ? 'bounce_rate' : null api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, this.state.query, {limit: 100, include: include}) .then((res) => this.setState({loading: false, pages: res})) } - showExtra() { + showBounceRate() { return this.state.query.period !== 'realtime' && !this.state.query.filters.goal } + showPageviews() { + const {filters} = this.state.query + return this.state.query.period !== 'realtime' && !(filters.goal || filters.source || filters.referrer) + } + formatBounceRate(page) { if (typeof(page.bounce_rate) === 'number') { return page.bounce_rate + '%' @@ -39,14 +44,19 @@ class PagesModal extends React.Component { {page.name} {numberFormatter(page.count)} - {this.showExtra() && {numberFormatter(page.unique_visitors)} } - {this.showExtra() && {this.formatBounceRate(page)} } + {this.showPageviews() && {numberFormatter(page.pageviews)} } + {this.showBounceRate() && {this.formatBounceRate(page)} } ) } label() { - return this.state.query.period === 'realtime' ? 'Active visitors' : 'Pageviews' + return this.state.query.period === 'realtime' ? 'Active visitors' : 'Visitors' + } + + title() { + const {filters} = this.state.query + return (filters.source || filters.referrer) ? 'Entry Pages' : 'Top Pages' } renderBody() { @@ -57,7 +67,7 @@ class PagesModal extends React.Component { } else if (this.state.pages) { return ( -

Top pages

+

{this.title()}

@@ -66,8 +76,8 @@ class PagesModal extends React.Component { Page url { this.label() } - {this.showExtra() && Unique visitors} - {this.showExtra() && Bounce rate} + {this.showPageviews() && Pageviews} + {this.showBounceRate() && Bounce rate} diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index 5dfa02dff..629e62972 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -48,12 +48,26 @@ class ReferrerDrilldownModal extends React.Component { } } + renderExternalLink(name) { + return ( + + + + ) + } + renderReferrerName(name) { - if (name) { - return {name} - } else { - return '(no referrer)' - } + const query = new URLSearchParams(window.location.search) + query.set('referrer', name) + + return ( + + + { name === '' ? '(no referrer)' : name } + + { this.renderExternalLink(name) } + + ) } renderTweet(tweet, index) { @@ -130,7 +144,7 @@ class ReferrerDrilldownModal extends React.Component { } else if (this.state.referrers) { return ( - ← All referrers +

Referrer drilldown

diff --git a/assets/js/dashboard/stats/modals/referrers.js b/assets/js/dashboard/stats/modals/referrers.js index e20c996b2..c431f4dd2 100644 --- a/assets/js/dashboard/stats/modals/referrers.js +++ b/assets/js/dashboard/stats/modals/referrers.js @@ -49,11 +49,14 @@ class ReferrersModal extends React.Component { } renderReferrer(referrer) { + const query = new URLSearchParams(window.location.search) + query.set('source', referrer.name) + return ( - { referrer.name } + { referrer.name } {numberFormatter(referrer.count)} {this.showExtra() && {this.formatBounceRate(referrer)} } @@ -74,7 +77,7 @@ class ReferrersModal extends React.Component { } else if (this.state.referrers) { return ( -

Top Referrers

+

Top Sources

diff --git a/assets/js/dashboard/stats/pages.js b/assets/js/dashboard/stats/pages.js index 00190c84b..49e704e5c 100644 --- a/assets/js/dashboard/stats/pages.js +++ b/assets/js/dashboard/stats/pages.js @@ -27,8 +27,14 @@ export default class Pages extends React.Component { } fetchPages() { - api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, this.props.query) - .then((res) => this.setState({loading: false, pages: res})) + const {filters} = this.props.query + if (filters.source || filters.referrer) { + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/entry-pages`, this.props.query) + .then((res) => this.setState({loading: false, pages: res})) + } else { + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/pages`, this.props.query) + .then((res) => this.setState({loading: false, pages: res})) + } } renderPage(page) { @@ -44,7 +50,7 @@ export default class Pages extends React.Component { } label() { - return this.props.query.period === 'realtime' ? 'Active visitors' : 'Pageviews' + return this.props.query.period === 'realtime' ? 'Active visitors' : 'Visitors' } renderList() { @@ -66,11 +72,16 @@ export default class Pages extends React.Component { } } + title() { + const filters = this.props.query.filters + return filters['source'] || filters['referrer'] ? 'Entry Pages' : 'Top Pages' + } + renderContent() { if (this.state.pages) { return ( -

Top Pages

+

{this.title()}

{ this.renderList() }
diff --git a/assets/js/dashboard/stats/referrers.js b/assets/js/dashboard/stats/referrers.js index f99ad0dbb..a3e66b869 100644 --- a/assets/js/dashboard/stats/referrers.js +++ b/assets/js/dashboard/stats/referrers.js @@ -8,6 +8,15 @@ import MoreLink from './more-link' import numberFormatter from '../number-formatter' import * as api from '../api' +function LinkOption(props) { + if (props.disabled) { + return {props.children} + } else { + props = Object.assign({}, props, {className: props.className + ' hover:underline'}) + return {props.children} + } +} + export default class Referrers extends React.Component { constructor(props) { super(props) @@ -27,7 +36,11 @@ export default class Referrers extends React.Component { } fetchReferrers() { - if (this.props.query.filters.goal) { + if (this.props.query.filters.source) { + api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/${this.props.query.filters.source}`, this.props.query) + .then((res) => res.search_terms || res.referrers) + .then((referrers) => this.setState({loading: false, referrers: referrers})) + } else if (this.props.query.filters.goal) { api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers`, this.props.query) .then((res) => this.setState({loading: false, referrers: res})) } else { @@ -36,15 +49,45 @@ export default class Referrers extends React.Component { } } + renderExternalLink(referrer) { + if (this.props.query.filters.source && this.props.query.filters.source !== 'Google') { + return ( + + + + ) + } + return null + } + + renderFavicon(referrer) { + if (referrer.url) { + return ( + + ) + } + } + renderReferrer(referrer) { + const query = new URLSearchParams(window.location.search) + + if (this.props.query.filters.source) { + query.set('referrer', referrer.name) + } else { + query.set('source', referrer.name) + } + return (
- - - { referrer.name } - + + + { this.renderFavicon(referrer) } + { referrer.name === '' ? '(no referrer)' : referrer.name } + + { this.renderExternalLink(referrer) } +
{numberFormatter(referrer.count)}
@@ -57,11 +100,15 @@ export default class Referrers extends React.Component { renderList() { if (this.state.referrers.length > 0) { + const source = this.props.query.filters.source + const keyLabel = source === 'Google' ? 'Search term' : source ? 'Referrer' : 'Source' + const valLabel = this.props.query.period === 'realtime' ? 'Active visitors' : 'Visitors' + return (
- Referrer - { this.label() } + { keyLabel } + { valLabel }
@@ -75,12 +122,16 @@ export default class Referrers extends React.Component { } renderContent() { + const source = this.props.query.filters.source + const title = source === 'Google' ? 'Search terms' : source ? 'Top Referrers' : 'Top Sources' + const endpoint = source ? 'referrers/' + source : 'referrers' + if (this.state.referrers) { return ( -

Top Referrers

+

{title}

{ this.renderList() } - +
) } diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 7844868b4..cdb85fc2a 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -1,7 +1,6 @@ import React from 'react'; import { withRouter } from 'react-router-dom' import Chart from 'chart.js' -import FadeIn from '../fade-in' import { eventName } from '../query' import numberFormatter, {durationFormatter} from '../number-formatter' import * as api from '../api' @@ -310,9 +309,7 @@ export default class VisitorGraph extends React.Component { return (
{ this.state.loading &&
} - - { this.renderInner() } - + { this.renderInner() }
) } diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex index a5214bf18..96201db31 100644 --- a/lib/plausible/billing/billing.ex +++ b/lib/plausible/billing/billing.ex @@ -125,8 +125,7 @@ defmodule Plausible.Billing do defp site_usage(site) do q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"}) - {pageviews, _} = Plausible.Stats.Clickhouse.pageviews_and_visitors(site, q) - pageviews + Plausible.Stats.Clickhouse.total_events(site, q) end defp format_subscription(params) do diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 2bcb50282..74a4dfd9a 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -41,7 +41,7 @@ defmodule Plausible.Google.Api do |> Enum.map(fn url -> String.trim_trailing(url, "/") end) end - def fetch_stats(auth, query) do + def fetch_stats(auth, query, limit) do auth = refresh_if_needed(auth) property = URI.encode_www_form(auth.property) @@ -52,7 +52,7 @@ defmodule Plausible.Google.Api do startDate: Date.to_iso8601(query.date_range.first), endDate: Date.to_iso8601(query.date_range.last), dimensions: ["query"], - rowLimit: 20 + rowLimit: limit }), "Content-Type": "application/json", Authorization: "Bearer #{auth.access_token}" diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index b11d8cba8..f7d17a858 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -111,12 +111,12 @@ defmodule Plausible.Stats.Clickhouse do {plot, labels, present_index} end - def calculate_plot(site, %Query{period: "realtime"}) do + def calculate_plot(site, %Query{period: "realtime"} = query) do + query = %Query{query | period: "30m"} + groups = Clickhouse.all( - from e in "events", - where: e.domain == ^site.domain, - where: e.timestamp >= fragment("now() - INTERVAL 31 MINUTE"), + from e in base_query(site, query), select: { fragment("dateDiff('minute', now(), ?) as relativeMinute", e.timestamp), fragment("count(*) as pageviews") @@ -133,14 +133,10 @@ defmodule Plausible.Stats.Clickhouse do end def bounce_rate(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - [res] = Clickhouse.all( - from s in "sessions", - select: {fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime + from s in base_session_query(site, query), + select: {fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")} ) res["bounce_rate"] || 0 @@ -156,23 +152,35 @@ defmodule Plausible.Stats.Clickhouse do res["visit_duration"] || 0 end - def total_pageviews(site, %Query{period: "realtime"}) do + def total_pageviews(site, %Query{period: "realtime"} = query) do + query = %Query{query | period: "30m"} + [res] = Clickhouse.all( - from e in "events", - select: fragment("count(*) as pageviews"), - where: e.timestamp >= fragment("now() - INTERVAL 30 MINUTE"), - where: e.domain == ^site.domain + from e in base_session_query(site, query), + select: fragment("sum(sign * pageviews) as pageviews") ) res["pageviews"] end - def pageviews_and_visitors(site, query) do + def total_events(site, query) do [res] = Clickhouse.all( from e in base_query(site, query), - select: {fragment("count(*) as pageviews"), fragment("uniq(user_id) as visitors")} + select: fragment("count(*) as events") + ) + + res["events"] + end + + def pageviews_and_visitors(site, query) do + [res] = + Clickhouse.all( + from e in base_session_query(site, query), + select: + {fragment("sum(sign * pageviews) as pageviews"), + fragment("uniq(user_id) as visitors")} ) {res["pageviews"], res["visitors"]} @@ -258,7 +266,9 @@ defmodule Plausible.Stats.Clickhouse do where: e.referrer_source == "" ) - if no_referrers |> hd |> Map.get("count") > 0, do: referrers ++ no_referrers, else: [] + if no_referrers |> hd |> Map.get("count") > 0, + do: referrers ++ no_referrers, + else: referrers else referrers end @@ -294,26 +304,16 @@ defmodule Plausible.Stats.Clickhouse do res["visitors"] end - def referrer_drilldown(site, query, referrer, include \\ []) do + def referrer_drilldown(site, query, referrer, include, limit) do referrer = if referrer == @no_ref, do: "", else: referrer - referring_urls = - Clickhouse.all( - from e in base_session_query(site, query), - select: {fragment("? as name", e.referrer), fragment("uniq(user_id) as count")}, - group_by: e.referrer, - where: e.referrer_source == ^referrer, - order_by: [desc: fragment("count")], - limit: 100 - ) - q = from( s in base_session_query(site, query), group_by: s.referrer, where: s.referrer_source == ^referrer, order_by: [desc: fragment("count")], - limit: 100 + limit: ^limit ) q = @@ -331,7 +331,12 @@ defmodule Plausible.Stats.Clickhouse do ) end - referring_urls = Clickhouse.all(q) + referring_urls = + Clickhouse.all(q) + |> Enum.map(fn ref -> + url = if ref["name"] !== "", do: URI.parse("http://" <> ref["name"]).host + Map.put(ref, "url", url) + end) if referrer == "Twitter" do urls = Enum.map(referring_urls, & &1["name"]) @@ -370,6 +375,24 @@ defmodule Plausible.Stats.Clickhouse do ) end + def entry_pages(site, query, limit, include) do + pages = Clickhouse.all( + from s in base_session_query(site, query), + group_by: s.entry_page, + order_by: [desc: fragment("count")], + limit: ^limit, + select: + {fragment("? as name", s.entry_page), fragment("uniq(?) as count", s.user_id)} + ) + + if "bounce_rate" in include do + bounce_rates = bounce_rates_by_page_url(site, query) + Enum.map(pages, fn url -> Map.put(url, "bounce_rate", bounce_rates[url["name"]]) end) + else + pages + end + end + def top_pages(site, %Query{period: "realtime"} = query, limit, _include) do Clickhouse.all( from s in base_session_query(site, query), @@ -386,23 +409,12 @@ defmodule Plausible.Stats.Clickhouse do e in base_query(site, query), group_by: e.pathname, order_by: [desc: fragment("count")], - limit: ^limit + limit: ^limit, + select: + {fragment("? as name", e.pathname), fragment("uniq(?) as count", e.user_id), + fragment("count(*) as pageviews")} ) - q = - if "unique_visitors" in include do - from( - e in q, - select: - {fragment("? as name", e.pathname), fragment("count(?) as count", e.pathname), - fragment("uniq(?) as unique_visitors", e.user_id)} - ) - else - from(e in q, - select: {fragment("? as name", e.pathname), fragment("count(?) as count", e.pathname)} - ) - end - pages = Clickhouse.all(q) if "bounce_rate" in include do @@ -414,15 +426,11 @@ defmodule Plausible.Stats.Clickhouse do end defp bounce_rates_by_page_url(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - Clickhouse.all( - from s in "sessions", + from s in base_session_query(site, query), select: {s.entry_page, fragment("count(*) as total"), fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime, group_by: s.entry_page, order_by: [desc: fragment("total")], limit: 100 @@ -498,13 +506,11 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.take(limit) end - def current_visitors(site) do + def current_visitors(site, query) do [res] = Clickhouse.all( - from e in "events", - select: fragment("uniq(user_id) as visitors"), - where: e.timestamp >= fragment("now() - INTERVAL 5 MINUTE"), - where: e.domain == ^site.domain + from s in base_query(site, query), + select: fragment("uniq(user_id) as visitors") ) res["visitors"] @@ -561,11 +567,33 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.filter(& &1) if Enum.count(events) > 0 do - Clickhouse.all( - from e in base_query(site, query, events), + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + q = + from( + e in "events", + where: e.domain == ^site.domain, + where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, + where: fragment("? IN tuple(?)", e.name, ^events), select: {e.name, fragment("uniq(user_id) as count")}, group_by: e.name - ) + ) + + q = + if query.filters["source"] do + filtered_sessions = + from(s in base_session_query(site, query), select: %{session_id: s.session_id}) + + from( + e in q, + join: cs in subquery(filtered_sessions), + on: e.session_id == cs.session_id + ) + else + q + end + + Clickhouse.all(q) else [] end @@ -577,14 +605,35 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.filter(& &1) if Enum.count(pages) > 0 do - Clickhouse.all( - from e in base_query(site, query), + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + q = + from( + e in "events", + where: e.domain == ^site.domain, + where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, + where: fragment("? IN tuple(?)", e.pathname, ^pages), + group_by: e.pathname, select: {fragment("concat('Visit ', ?) as name", e.pathname), - fragment("uniq(user_id) as count")}, - where: fragment("? IN tuple(?)", e.pathname, ^pages), - group_by: e.pathname - ) + fragment("uniq(user_id) as count")} + ) + + q = + if query.filters["source"] do + filtered_sessions = + from(s in base_session_query(site, query), select: %{session_id: s.session_id}) + + from( + e in q, + join: cs in subquery(filtered_sessions), + on: e.session_id == cs.session_id + ) + else + q + end + + Clickhouse.all(q) else [] end @@ -594,26 +643,32 @@ defmodule Plausible.Stats.Clickhouse do Enum.sort_by(conversions, fn conversion -> -conversion["count"] end) end - defp base_session_query(site, %Query{period: "realtime"}) do - first_datetime = - Timex.now(site.timezone) |> Timex.shift(minutes: -5) |> Timex.Timezone.convert("UTC") - - from(s in "sessions", - where: s.domain == ^site.domain, - where: s.timestamp >= ^first_datetime - ) - end - defp base_session_query(site, query) do {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - from(s in "sessions", - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime - ) + q = + from(s in "sessions", + where: s.domain == ^site.domain, + where: s.start >= ^first_datetime and s.start < ^last_datetime + ) + + q = + if query.filters["source"] do + source = query.filters["source"] + from(e in q, where: e.referrer_source == ^source) + else + q + end + + if query.filters["referrer"] do + ref = query.filters["referrer"] + from(e in q, where: e.referrer == ^ref) + else + q + end end - defp base_query(site, query, events \\ ["pageview"]) do + defp base_query(site, query) do {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) {goal_event, path} = event_name_for_goal(query) @@ -623,6 +678,22 @@ defmodule Plausible.Stats.Clickhouse do where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) + q = + if query.filters["source"] do + source = query.filters["source"] + from(e in q, where: e.referrer_source == ^source) + else + q + end + + q = + if query.filters["referrer"] do + ref = query.filters["referrer"] + from(e in q, where: e.referrer == ^ref) + else + q + end + q = if path do from(e in q, where: e.pathname == ^path) @@ -633,7 +704,7 @@ defmodule Plausible.Stats.Clickhouse do if goal_event do from(e in q, where: e.name == ^goal_event) else - from(e in q, where: fragment("? IN tuple(?)", e.name, ^events)) + from(e in q, where: e.name == "pageview") end end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index a9d282d7f..1818bce4a 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -13,13 +13,14 @@ defmodule Plausible.Stats.Query do Map.put(query, :date_range, Date.range(new_first, new_last)) end - def from(tz, %{"period" => "realtime"}) do + def from(tz, %{"period" => "realtime"} = params) do date = today(tz) %__MODULE__{ period: "realtime", step_type: "minute", - date_range: Date.range(date, date) + date_range: Date.range(date, date), + filters: parse_filters(params) } end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 331eae7ac..a5326302e 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -26,7 +26,7 @@ defmodule PlausibleWeb.Api.StatsController do [ %{ name: "Active visitors", - count: Stats.current_visitors(site) + count: Stats.current_visitors(site, query) }, %{ name: "Pageviews (last 30 min)", @@ -138,7 +138,7 @@ defmodule PlausibleWeb.Api.StatsController do search_terms = if site.google_auth && site.google_auth.property && !query.filters["goal"] do - @google_api.fetch_stats(site.google_auth, query) + @google_api.fetch_stats(site.google_auth, query, params["limit"] || 9) end case search_terms do @@ -162,8 +162,9 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) include = if params["include"], do: String.split(params["include"], ","), else: [] + limit = params["limit"] || 9 - referrers = Stats.referrer_drilldown(site, query, referrer, include) + referrers = Stats.referrer_drilldown(site, query, referrer, include, limit) total_visitors = Stats.visitors_from_referrer(site, query, referrer) json(conn, %{referrers: referrers, total_visitors: total_visitors}) end @@ -186,6 +187,15 @@ defmodule PlausibleWeb.Api.StatsController do json(conn, Stats.top_pages(site, query, limit || 9, include)) end + def entry_pages(conn, params) do + site = conn.assigns[:site] + query = Query.from(site.timezone, params) + include = if params["include"], do: String.split(params["include"], ","), else: [] + limit = if params["limit"], do: String.to_integer(params["limit"]) + + json(conn, Stats.entry_pages(site, query, limit || 9, include)) + end + def countries(conn, params) do site = conn.assigns[:site] query = Query.from(site.timezone, params) @@ -197,14 +207,14 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) - json(conn, Stats.browsers(site, query, parse_integer(params["limit"]) || 9)) + json(conn, Stats.browsers(site, query, params["limit"] || 9)) end def operating_systems(conn, params) do site = conn.assigns[:site] query = Query.from(site.timezone, params) - json(conn, Stats.operating_systems(site, query, parse_integer(params["limit"]) || 9)) + json(conn, Stats.operating_systems(site, query, params["limit"] || 9)) end def screen_sizes(conn, params) do @@ -222,15 +232,8 @@ defmodule PlausibleWeb.Api.StatsController do end def current_visitors(conn, _) do - json(conn, Stats.current_visitors(conn.assigns[:site])) - end - - defp parse_integer(nil), do: nil - - defp parse_integer(nr) do - case Integer.parse(nr) do - {number, ""} -> number - _ -> nil - end + site = conn.assigns[:site] + query = Query.from(site.timezone, %{"period" => "realtime"}) + json(conn, Stats.current_visitors(site, query)) end end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index a48eb00a3..bc2a87a0c 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -47,6 +47,7 @@ defmodule PlausibleWeb.Router do get "/:domain/referrers/:referrer", StatsController, :referrer_drilldown get "/:domain/goal/referrers/:referrer", StatsController, :referrer_drilldown_for_goal get "/:domain/pages", StatsController, :pages + get "/:domain/entry-pages", StatsController, :entry_pages get "/:domain/countries", StatsController, :countries get "/:domain/browsers", StatsController, :browsers get "/:domain/operating-systems", StatsController, :operating_systems diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index cddeab1a0..4d9820b29 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -83,5 +83,12 @@ defmodule Plausible.Stats.QueryTest do assert q.filters["goal"] == "Signup" end + + test "parses source filter" do + filters = Jason.encode!(%{"source" => "Twitter"}) + q = Query.from(@tz, %{"period" => "6mo", "filters" => filters}) + + assert q.filters["source"] == "Twitter" + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 72467e0d0..226326a25 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -83,14 +83,14 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01") res = json_response(conn, 200) - assert %{"name" => "Unique visitors", "count" => 6, "change" => 100} in res["top_stats"] + assert %{"name" => "Unique visitors", "count" => 3, "change" => 100} in res["top_stats"] end test "counts total pageviews", %{conn: conn, site: site} do conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01") res = json_response(conn, 200) - assert %{"name" => "Total pageviews", "count" => 6, "change" => 100} in res["top_stats"] + assert %{"name" => "Total pageviews", "count" => 3, "change" => 100} in res["top_stats"] end test "calculates bounce rate", %{conn: conn, site: site} do diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 73a178934..7e3118904 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -5,53 +5,86 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do describe "GET /api/stats/:domain/pages" do setup [:create_user, :log_in, :create_site] - test "returns top pages sources by pageviews", %{conn: conn, site: site} do + test "returns top pages by visitors", %{conn: conn, site: site} do conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "/"}, - %{"count" => 2, "name" => "/register"}, - %{"count" => 1, "name" => "/contact"}, - %{"count" => 1, "name" => "/irrelevant"} + %{"count" => 2, "pageviews" => 2, "name" => "/"}, + %{"count" => 2, "pageviews" => 2, "name" => "/register"}, + %{"count" => 1, "pageviews" => 1, "name" => "/contact"}, + %{"count" => 1, "pageviews" => 1, "name" => "/irrelevant"} ] end - test "calculates bounce rate and unique visitors for pages", %{conn: conn, site: site} do + test "calculates bounce rate for pages", %{conn: conn, site: site} do conn = get( conn, - "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&include=bounce_rate,unique_visitors" + "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&include=bounce_rate" ) assert json_response(conn, 200) == [ - %{"bounce_rate" => 33.0, "count" => 2, "unique_visitors" => 2, "name" => "/"}, - %{ - "bounce_rate" => nil, - "count" => 2, - "unique_visitors" => 2, - "name" => "/register" - }, - %{ - "bounce_rate" => nil, - "count" => 1, - "unique_visitors" => 1, - "name" => "/contact" - }, - %{ - "bounce_rate" => nil, - "count" => 1, - "unique_visitors" => 1, - "name" => "/irrelevant" - } - ] + %{ + "bounce_rate" => 33.0, + "count" => 2, + "pageviews" => 2, + "name" => "/"}, + %{ + "bounce_rate" => nil, + "count" => 2, + "pageviews" => 2, + "name" => "/register" + }, + %{ + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/contact" + }, + %{ + "bounce_rate" => nil, + "count" => 1, + "pageviews" => 1, + "name" => "/irrelevant" + } + ] end test "returns top pages in realtime report", %{conn: conn, site: site} do conn = get(conn, "/api/stats/#{site.domain}/pages?period=realtime") assert json_response(conn, 200) == [ - %{"count" => 3, "name" => "/"} + %{"count" => 2, "name" => "/exit"}, + %{"count" => 1, "name" => "/"} + ] + end + end + + describe "GET /api/stats/:domain/entry-pages" do + setup [:create_user, :log_in, :create_site] + + test "returns top entry pages by visitors", %{conn: conn, site: site} do + conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2019-01-01") + + assert json_response(conn, 200) == [ + %{"count" => 3, "name" => "/"}, ] end + + test "calculates bounce rate for entry pages", %{conn: conn, site: site} do + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2019-01-01&include=bounce_rate" + ) + + assert json_response(conn, 200) == [ + %{ + "bounce_rate" => 33.0, + "count" => 3, + "name" => "/" + } + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index ec22f17df..273536579 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -44,7 +44,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do assert json_response(conn, 200) == [ %{"name" => "10words", "count" => 2, "url" => "10words.com"}, - %{"name" => "Bing", "count" => 1, "url" => ""} + %{"name" => "Bing", "count" => 1, "url" => "bing.com"} ] end end @@ -90,7 +90,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do assert json_response(conn, 200) == %{ "total_visitors" => 2, "referrers" => [ - %{"name" => "10words.com/page1", "count" => 2} + %{"name" => "10words.com/page1", "url" => "10words.com", "count" => 2} ] } end @@ -107,6 +107,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do "referrers" => [ %{ "name" => "10words.com/page1", + "url" => "10words.com", "count" => 2, "bounce_rate" => 50.0, "visit_duration" => 50.0 @@ -118,7 +119,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do test "gets keywords from Google", %{conn: conn, user: user, site: site} do insert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com") conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day&date=2019-02-01") - {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil) + {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil) assert json_response(conn, 200) == %{ "total_visitors" => 2, diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex index b591186d2..ff6d0e593 100644 --- a/test/support/clickhouse_setup.ex +++ b/test/support/clickhouse_setup.ex @@ -287,6 +287,33 @@ defmodule Plausible.Test.ClickhouseSetup do referrer_source: "Twitter", start: ~N[2019-03-01 02:00:00], timestamp: ~N[2019-03-01 02:00:00] + }, + %{ + domain: "test-site.com", + entry_page: "/", + exit_page: "/", + referrer_source: "Bing", + referrer: "bing.com", + start: Timex.now() |> Timex.shift(minutes: -1), + timestamp: Timex.now() |> Timex.shift(minutes: -1) + }, + %{ + domain: "test-site.com", + entry_page: "/", + exit_page: "/exit", + referrer_source: "10words", + referrer: "10words.com", + start: Timex.now() |> Timex.shift(minutes: -2), + timestamp: Timex.now() |> Timex.shift(minutes: -2) + }, + %{ + domain: "test-site.com", + entry_page: "/", + exit_page: "/exit", + referrer_source: "10words", + referrer: "10words.com", + start: Timex.now() |> Timex.shift(minutes: -3), + timestamp: Timex.now() |> Timex.shift(minutes: -3) } ]) end diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.ex index 2e4e922f8..5aa33a718 100644 --- a/test/support/google_api_mock.ex +++ b/test/support/google_api_mock.ex @@ -1,5 +1,5 @@ defmodule Plausible.Google.Api.Mock do - def fetch_stats(_auth, _query) do + def fetch_stats(_auth, _query, _limit) do {:ok, [ %{"name" => "simple web analytics", "count" => 6},