From 96abfc4ea18476469b676569aea08f6f22464c1b Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 13:48:27 +0800 Subject: [PATCH 1/7] Click on goal to see conversions for it --- assets/js/dashboard/api.js | 3 +- assets/js/dashboard/filters.js | 23 +++++ assets/js/dashboard/index.js | 4 +- assets/js/dashboard/mount.js | 1 + assets/js/dashboard/query.js | 35 ++++--- assets/js/dashboard/router.js | 21 ++-- assets/js/dashboard/stats/browsers.js | 4 +- assets/js/dashboard/stats/conversions.js | 6 +- assets/js/dashboard/stats/countries.js | 4 +- .../dashboard/stats/modals/google-keywords.js | 10 +- .../stats/modals/referrer-drilldown.js | 14 ++- assets/js/dashboard/stats/modals/referrers.js | 2 +- .../js/dashboard/stats/operating-systems.js | 4 +- assets/js/dashboard/stats/pages.js | 7 +- assets/js/dashboard/stats/referrers.js | 6 +- assets/js/dashboard/stats/screen-sizes.js | 4 +- assets/js/dashboard/stats/visitor-graph.js | 3 +- assets/package-lock.json | 5 + assets/package.json | 3 +- lib/plausible/stats/query.ex | 59 ++++++------ lib/plausible/stats/stats.ex | 96 +++++++++++-------- .../controllers/api/stats_controller.ex | 2 +- test/plausible/stats/query_test.exs | 19 ++-- .../api/stats_controller/referrers_test.exs | 10 +- 24 files changed, 216 insertions(+), 129 deletions(-) create mode 100644 assets/js/dashboard/filters.js diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.js index 493bec9e1..6606020e7 100644 --- a/assets/js/dashboard/api.js +++ b/assets/js/dashboard/api.js @@ -11,7 +11,8 @@ function serialize(obj) { export function get(url, query, ...extraQuery) { query = Object.assign({}, query, { - date: query.date ? formatISO(query.date) : undefined + date: query.date ? formatISO(query.date) : undefined, + filters: query.filters ? JSON.stringify(query.filters) : undefined }, ...extraQuery) url = url + `?${serialize(query)}` diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js new file mode 100644 index 000000000..a76d7a141 --- /dev/null +++ b/assets/js/dashboard/filters.js @@ -0,0 +1,23 @@ +import React from 'react'; +import { withRouter } from 'react-router-dom' +import {removeQueryParam} from './query' + +function Filters({query, history, location}) { + if (query.filters.goal) { + function removeGoal() { + history.push({search: removeQueryParam(location.search, 'goal')}) + } + + return ( +
+ + Completed goal {query.filters.goal} ✕ + +
+ ) + } + + return null +} + +export default withRouter(Filters) diff --git a/assets/js/dashboard/index.js b/assets/js/dashboard/index.js index 06b8f8d57..9be7f8670 100644 --- a/assets/js/dashboard/index.js +++ b/assets/js/dashboard/index.js @@ -2,6 +2,7 @@ import React from 'react'; import { withRouter } from 'react-router-dom' import Datepicker from './datepicker' +import Filters from './filters' import CurrentVisitors from './stats/current-visitors' import VisitorGraph from './stats/visitor-graph' import Referrers from './stats/referrers' @@ -40,11 +41,12 @@ class Stats extends React.Component {
-

Analytics for {this.props.site.domain}

+

Analytics for {this.props.site.domain}

+
diff --git a/assets/js/dashboard/mount.js b/assets/js/dashboard/mount.js index b041d4cb2..b62775355 100644 --- a/assets/js/dashboard/mount.js +++ b/assets/js/dashboard/mount.js @@ -1,5 +1,6 @@ import React from 'react'; import ReactDOM from 'react-dom'; +import 'url-search-params-polyfill'; import Router from './router' diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index a117e8ed0..49070bcdd 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -1,19 +1,10 @@ import {formatDay, formatMonthYYYY, newDateInOffset} from './date' -function parseQueryString(queryString) { - var query = {}; - var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&'); - for (var i = 0; i < pairs.length; i++) { - var pair = pairs[i].split('='); - query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || ''); - } - return query; -} - const PERIODS = ['day', 'month', '7d', '30d', '3mo', '6mo'] export function parseQuery(querystring, site) { - let {period, date} = parseQueryString(querystring) + const q = new URLSearchParams(querystring) + let period = q.get('period') const periodKey = 'period__' + site.domain if (PERIODS.includes(period)) { @@ -28,7 +19,10 @@ export function parseQuery(querystring, site) { return { period: period, - date: date ? new Date(date) : newDateInOffset(site.offset) + date: q.get('date') ? new Date(q.get('date')) : newDateInOffset(site.offset), + filters: { + 'goal': q.get('goal') + } } } @@ -47,3 +41,20 @@ export function toHuman(query) { return 'in the last 6 months' } } + +export function removeQueryParam(search, parameter) { + const q = new URLSearchParams(search) + q.delete(parameter) + return q.toString() +} + +export function eventName(query) { + if (query.filters.goal) { + if (query.filters.goal.startsWith('Visit ')) { + return 'pageviews' + } + return 'events' + } else { + return 'pageviews' + } +} diff --git a/assets/js/dashboard/router.js b/assets/js/dashboard/router.js index 5bcc110e1..b4688c2d5 100644 --- a/assets/js/dashboard/router.js +++ b/assets/js/dashboard/router.js @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import Dash from './index' import Modal from './stats/modals/modal' import ReferrersModal from './stats/modals/referrers' @@ -9,16 +9,25 @@ import CountriesModal from './stats/modals/countries' import BrowsersModal from './stats/modals/browsers' import OperatingSystemsModal from './stats/modals/operating-systems' -import { - BrowserRouter, - Switch, - Route -} from "react-router-dom"; +import {BrowserRouter, Switch, Route, useLocation} from "react-router-dom"; + +function ScrollToTop() { + const location = useLocation(); + + useEffect(() => { + if (location.state && location.state.scrollTop) { + window.scrollTo(0, 0); + } + }, [location]); + + return null; +} export default function Router({site}) { return ( + diff --git a/assets/js/dashboard/stats/browsers.js b/assets/js/dashboard/stats/browsers.js index cf0feeba9..ae7d22438 100644 --- a/assets/js/dashboard/stats/browsers.js +++ b/assets/js/dashboard/stats/browsers.js @@ -41,13 +41,13 @@ export default class Browsers extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.browsers) { return ( -
+

Browsers

by visitors
diff --git a/assets/js/dashboard/stats/conversions.js b/assets/js/dashboard/stats/conversions.js index 120d6cb38..9148d3d1c 100644 --- a/assets/js/dashboard/stats/conversions.js +++ b/assets/js/dashboard/stats/conversions.js @@ -1,4 +1,5 @@ import React from 'react'; +import { Link } from 'react-router-dom' import Bar from './bar' import MoreLink from './more-link' @@ -28,10 +29,13 @@ export default class Conversions extends React.Component { } renderGoal(goal) { + const query = new URLSearchParams(window.location.search) + query.set('goal', goal.name) + return (
- { goal.name } + { goal.name } {numberFormatter(goal.count)}
diff --git a/assets/js/dashboard/stats/countries.js b/assets/js/dashboard/stats/countries.js index c2dea60f8..04bd1bd8e 100644 --- a/assets/js/dashboard/stats/countries.js +++ b/assets/js/dashboard/stats/countries.js @@ -43,13 +43,13 @@ export default class Countries extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.countries) { return ( -
+

Top Countries

by visitors
diff --git a/assets/js/dashboard/stats/modals/google-keywords.js b/assets/js/dashboard/stats/modals/google-keywords.js index 750249474..8304a8fc3 100644 --- a/assets/js/dashboard/stats/modals/google-keywords.js +++ b/assets/js/dashboard/stats/modals/google-keywords.js @@ -41,7 +41,15 @@ class GoogleKeywordsModal extends React.Component { } renderKeywords() { - if (this.state.notConfigured) { + if (this.state.query.filters.goal) { + return ( +
+ +
Sorry, we cannot show which keywords converted best for goal {this.state.query.filters.goal}
+
Google has a monopoly on that data which helps them dominate the analytics market
+
+ ) + } else if (this.state.notConfigured) { if (this.state.isOwner) { return (
diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index 1dfcf82ff..ad38fce6e 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -33,6 +33,14 @@ class ReferrerDrilldownModal extends React.Component { ) } + renderGoalText() { + if (this.state.query.filters.goal) { + return ( +

completed {this.state.query.filters.goal}

+ ) + } + } + renderBody() { if (this.state.loading) { return ( @@ -47,10 +55,10 @@ class ReferrerDrilldownModal extends React.Component {
-

{this.state.totalVisitors} new visitors from {this.props.match.params.referrer}

-

{toHuman(this.state.query)}

+

{this.state.totalVisitors} visitors from {this.props.match.params.referrer}
{toHuman(this.state.query)}

+ {this.renderGoalText()} -
+
{ this.state.referrers.map(this.renderReferrer.bind(this)) }
diff --git a/assets/js/dashboard/stats/modals/referrers.js b/assets/js/dashboard/stats/modals/referrers.js index 84cc9630e..e00b551d1 100644 --- a/assets/js/dashboard/stats/modals/referrers.js +++ b/assets/js/dashboard/stats/modals/referrers.js @@ -43,7 +43,7 @@ class ReferrersModal extends React.Component {

Referrers

-
by new visitors
+
by visitors
diff --git a/assets/js/dashboard/stats/operating-systems.js b/assets/js/dashboard/stats/operating-systems.js index 3cc1ce583..e07bc7af9 100644 --- a/assets/js/dashboard/stats/operating-systems.js +++ b/assets/js/dashboard/stats/operating-systems.js @@ -41,13 +41,13 @@ export default class OperatingSystems extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.systems) { return ( -
+

Operating Systems

by visitors
diff --git a/assets/js/dashboard/stats/pages.js b/assets/js/dashboard/stats/pages.js index 540fec7dd..737bf1577 100644 --- a/assets/js/dashboard/stats/pages.js +++ b/assets/js/dashboard/stats/pages.js @@ -3,6 +3,7 @@ import React from 'react'; import Bar from './bar' import MoreLink from './more-link' import numberFormatter from '../number-formatter' +import { eventName } from '../query' import * as api from '../api' export default class Pages extends React.Component { @@ -44,16 +45,16 @@ export default class Pages extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.pages) { return ( -
+

Top Pages

-
by pageviews
+
by {eventName(this.props.query)}
diff --git a/assets/js/dashboard/stats/referrers.js b/assets/js/dashboard/stats/referrers.js index c2b073ba2..7e1a2d63b 100644 --- a/assets/js/dashboard/stats/referrers.js +++ b/assets/js/dashboard/stats/referrers.js @@ -43,16 +43,16 @@ export default class Referrers extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.referrers) { return ( -
+

Top Referrers

-
by new visitors
+
by visitors
diff --git a/assets/js/dashboard/stats/screen-sizes.js b/assets/js/dashboard/stats/screen-sizes.js index c3ea74d18..075cfbb3d 100644 --- a/assets/js/dashboard/stats/screen-sizes.js +++ b/assets/js/dashboard/stats/screen-sizes.js @@ -70,13 +70,13 @@ export default class ScreenSizes extends React.Component { render() { if (this.state.loading) { return ( -
+
) } else if (this.state.sizes) { return ( -
+

Screen Sizes

by visitors
diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 7275b094f..cb2eb087b 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -1,6 +1,7 @@ import React from 'react'; import { withRouter } from 'react-router-dom' import Chart from 'chart.js' +import { eventName } from '../query' import numberFormatter from '../number-formatter' import { isToday, shiftMonths, formatMonth } from '../date' import * as api from '../api' @@ -205,7 +206,7 @@ class LineGraph extends React.Component { {this.renderComparison(graphData.change_visitors)}
-
TOTAL PAGEVIEWS
+
TOTAL {eventName(this.props.query)}
{numberFormatter(graphData.pageviews)}
diff --git a/assets/package-lock.json b/assets/package-lock.json index 91d2842dc..de73f6249 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -9444,6 +9444,11 @@ } } }, + "url-search-params-polyfill": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/url-search-params-polyfill/-/url-search-params-polyfill-7.0.0.tgz", + "integrity": "sha512-0SEH3s+wCNbxEE/rWUalN004ICNi23Q74Ksc0gS2kG8EXnbayxGOrV97JdwnIVPKZ75Xk0hvKXvtIC4xReLMgg==" + }, "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", diff --git a/assets/package.json b/assets/package.json index 337a392e9..3aabe4c1c 100644 --- a/assets/package.json +++ b/assets/package.json @@ -12,7 +12,8 @@ "phoenix_html": "file:../deps/phoenix_html", "react": "^16.11.0", "react-dom": "^16.11.0", - "react-router-dom": "^5.1.2" + "react-router-dom": "^5.1.2", + "url-search-params-polyfill": "^7.0.0" }, "devDependencies": { "@babel/core": "^7.5.5", diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index c5b779ace..7ae9d28cb 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,5 +1,5 @@ defmodule Plausible.Stats.Query do - defstruct [date_range: nil, step_type: nil, period: nil, steps: nil] + defstruct [date_range: nil, step_type: nil, period: nil, steps: nil, filters: %{}] def new(attrs) do attrs @@ -31,61 +31,66 @@ defmodule Plausible.Stats.Query do Map.put(query, :date_range, Date.range(new_first, new_last)) end - def from(_tz, %{"period" => "day", "date" => date}) do + def from(_tz, %{"period" => "day", "date" => date} = params) do date = Date.from_iso8601!(date) %__MODULE__{ period: "day", date_range: Date.range(date, date), - step_type: "hour" + step_type: "hour", + filters: parse_filters(params) } end - def from(tz, %{"period" => "day"}) do + def from(tz, %{"period" => "day"} = params) do date = today(tz) %__MODULE__{ period: "day", date_range: Date.range(date, date), - step_type: "hour" + step_type: "hour", + filters: parse_filters(params) } end - def from(tz, %{"period" => "7d"}) do + def from(tz, %{"period" => "7d"} = params) do end_date = today(tz) start_date = end_date |> Timex.shift(days: -7) %__MODULE__{ period: "7d", date_range: Date.range(start_date, end_date), - step_type: "date" + step_type: "date", + filters: parse_filters(params) } end - def from(tz, %{"period" => "30d"}) do + def from(tz, %{"period" => "30d"} = params) do end_date = today(tz) start_date = end_date |> Timex.shift(days: -30) %__MODULE__{ period: "30d", date_range: Date.range(start_date, end_date), - step_type: "date" + step_type: "date", + filters: parse_filters(params) } end - def from(_tz, %{"period" => "month", "date" => month_start}) do - start_date = Date.from_iso8601!(month_start) |> Timex.beginning_of_month + def from(_tz, %{"period" => "month", "date" => date} = params) do + start_date = Date.from_iso8601!(date) |> Timex.beginning_of_month end_date = Timex.end_of_month(start_date) %__MODULE__{ period: "month", date_range: Date.range(start_date, end_date), step_type: "date", - steps: Timex.diff(start_date, end_date, :days) + steps: Timex.diff(start_date, end_date, :days), + filters: parse_filters(params) } end - def from(tz, %{"period" => "3mo"}) do + def from(tz, %{"period" => "3mo"} = params) do start_date = Timex.shift(today(tz), months: -2) |> Timex.beginning_of_month() @@ -93,11 +98,12 @@ defmodule Plausible.Stats.Query do period: "3mo", date_range: Date.range(start_date, today(tz)), step_type: "month", - steps: 3 + steps: 3, + filters: parse_filters(params) } end - def from(tz, %{"period" => "6mo"}) do + def from(tz, %{"period" => "6mo"} = params) do start_date = Timex.shift(today(tz), months: -5) |> Timex.beginning_of_month() @@ -105,28 +111,23 @@ defmodule Plausible.Stats.Query do period: "6mo", date_range: Date.range(start_date, today(tz)), step_type: "month", - steps: 6 - } - end - - def from(_tz, %{"period" => "custom", "from" => from, "to" => to}) do - start_date = Date.from_iso8601!(from) - end_date = Date.from_iso8601!(to) - date_range = Date.range(start_date, end_date) - - %__MODULE__{ - period: "custom", - date_range: date_range, - step_type: "date" + steps: 6, + filters: parse_filters(params) } end def from(tz, _) do - __MODULE__.from(tz, %{"period" => "6mo"}) + __MODULE__.from(tz, %{"period" => "30d"}) end defp today(tz) do Timex.now(tz) |> Timex.to_date end + + defp parse_filters(params) do + if params["filters"] do + Jason.decode!(params["filters"]) + end + end end diff --git a/lib/plausible/stats/stats.ex b/lib/plausible/stats/stats.ex index 3ad3709da..2c81a2245 100644 --- a/lib/plausible/stats/stats.ex +++ b/lib/plausible/stats/stats.ex @@ -96,9 +96,9 @@ defmodule Plausible.Stats do def top_referrers(site, query, limit \\ 5) do Repo.all(from e in base_query(site, query), - select: %{name: e.referrer_source, count: count(e.referrer_source)}, + select: %{name: e.referrer_source, count: count(e.user_id, :distinct)}, group_by: e.referrer_source, - where: e.new_visitor == true and not is_nil(e.referrer_source), + where: not is_nil(e.referrer_source), order_by: [desc: 2], limit: ^limit ) @@ -107,16 +107,16 @@ defmodule Plausible.Stats do def visitors_from_referrer(site, query, referrer) do Repo.one( from e in base_query(site, query), - select: count(e), - where: e.new_visitor == true and e.referrer_source == ^referrer + select: count(e.user_id, :distinct), + where: e.referrer_source == ^referrer ) end def referrer_drilldown(site, query, referrer) do Repo.all(from e in base_query(site, query), - select: %{name: e.referrer, count: count(e)}, + select: %{name: e.referrer, count: count(e.user_id, :distinct)}, group_by: e.referrer, - where: e.new_visitor == true and e.referrer_source == ^referrer, + where: e.referrer_source == ^referrer, order_by: [desc: 2], limit: 100 ) @@ -135,9 +135,9 @@ defmodule Plausible.Stats do def top_screen_sizes(site, query) do Repo.all(from e in base_query(site, query), - select: {e.screen_size, count(e.screen_size)}, + select: {e.screen_size, count(e.user_id, :distinct)}, group_by: e.screen_size, - where: e.new_visitor == true and not is_nil(e.screen_size) + where: not is_nil(e.screen_size) ) |> Enum.sort(fn {screen_size1, _}, {screen_size2, _} -> index1 = Enum.find_index(@available_screen_sizes, fn s -> s == screen_size1 end) @@ -160,10 +160,10 @@ defmodule Plausible.Stats do def countries(site, query, limit \\ 5) do Repo.all(from e in base_query(site, query), - select: {e.country_code, count(e.country_code)}, + select: {e.country_code, count(e.user_id, :distinct)}, group_by: e.country_code, - where: e.new_visitor == true and not is_nil(e.country_code), - order_by: [desc: count(e.country_code)] + where: not is_nil(e.country_code), + order_by: [desc: 2] ) |> Enum.map(fn {country_code, count} -> {Plausible.Stats.CountryName.from_iso3166(country_code), count} @@ -174,10 +174,10 @@ defmodule Plausible.Stats do def browsers(site, query, limit \\ 5) do Repo.all(from e in base_query(site, query), - select: {e.browser, count(e.browser)}, + select: {e.browser, count(e.user_id, :distinct)}, group_by: e.browser, - where: e.new_visitor == true and not is_nil(e.browser), - order_by: [desc: count(e.browser)] + where: not is_nil(e.browser), + order_by: [desc: 2] ) |> add_percentages |> Enum.take(limit) @@ -185,10 +185,10 @@ defmodule Plausible.Stats do def operating_systems(site, query, limit \\ 5) do Repo.all(from e in base_query(site, query), - select: {e.operating_system, count(e.operating_system)}, + select: {e.operating_system, count(e.user_id, :distinct)}, group_by: e.operating_system, - where: e.new_visitor == true and not is_nil(e.operating_system), - order_by: [desc: count(e.operating_system)] + where: not is_nil(e.operating_system), + order_by: [desc: 2] ) |> add_percentages |> Enum.take(limit) @@ -203,7 +203,15 @@ defmodule Plausible.Stats do ) end - def goal_conversions(site, query, _limit \\ 5) do + def goal_conversions(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do + Repo.all(from e in base_query(site, query), + select: %{name: ^goal, count: count(e.user_id, :distinct)}, + group_by: e.name, + order_by: [desc: 2] + ) + end + + def goal_conversions(site, query) do goals = Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain) fetch_pageview_goals(goals, site, query) ++ fetch_event_goals(goals, site, query) @@ -211,21 +219,12 @@ defmodule Plausible.Stats do end defp fetch_event_goals(goals, site, query) do - {:ok, first} = NaiveDateTime.new(query.date_range.first, ~T[00:00:00]) - first_datetime = Timex.to_datetime(first, site.timezone) - - {:ok, last} = NaiveDateTime.new(query.date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) - last_datetime = Timex.to_datetime(last, site.timezone) - events = Enum.map(goals, fn goal -> goal.event_name end) |> Enum.filter(&(&1)) if Enum.count(events) > 0 do Repo.all( - from e in Plausible.Event, - where: e.hostname == ^site.domain, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, - where: e.name in ^events, + from e in base_query(site, query, events), group_by: e.name, select: %{name: e.name, count: count(e.user_id, :distinct)} ) @@ -235,21 +234,12 @@ defmodule Plausible.Stats do end defp fetch_pageview_goals(goals, site, query) do - {:ok, first} = NaiveDateTime.new(query.date_range.first, ~T[00:00:00]) - first_datetime = Timex.to_datetime(first, site.timezone) - - {:ok, last} = NaiveDateTime.new(query.date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) - last_datetime = Timex.to_datetime(last, site.timezone) - pages = Enum.map(goals, fn goal -> goal.page_path end) |> Enum.filter(&(&1)) if Enum.count(pages) > 0 do Repo.all( - from e in Plausible.Event, - where: e.hostname == ^site.domain, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime, - where: e.name == "pageview", + from e in base_query(site, query), where: e.pathname in ^pages, group_by: e.pathname, select: %{name: fragment("concat('Visit ', ?)", e.pathname), count: count(e.user_id, :distinct)} @@ -263,7 +253,7 @@ defmodule Plausible.Stats do Enum.sort_by(conversions, fn conversion -> -conversion[:count] end) end - defp base_query(site, query) do + defp base_query(site, query, events \\ ["pageview"]) do {:ok, first} = NaiveDateTime.new(query.date_range.first, ~T[00:00:00]) first_datetime = Timex.to_datetime(first, site.timezone) |> Timex.Timezone.convert("UTC") @@ -272,11 +262,35 @@ defmodule Plausible.Stats do last_datetime = Timex.to_datetime(last, site.timezone) |> Timex.Timezone.convert("UTC") - from(e in Plausible.Event, - where: e.name == "pageview", + {goal_event, path} = event_name_for_goal(query) + + q = from(e in Plausible.Event, where: e.hostname == ^site.domain, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) + + q = if path do + from(e in q, where: e.pathname == ^path) + else + q + end + + if goal_event do + from(e in q, where: e.name == ^goal_event) + else + from(e in q, where: e.name in ^events) + end + end + + defp event_name_for_goal(query) do + case query.filters["goal"] do + "Visit " <> page -> + {"pageview", page} + goal when is_binary(goal) -> + {goal, nil} + _ -> + {nil, nil} + end end defp transform_keys(map, fun) do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 8ac28b86b..af48b7d75 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -36,7 +36,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] |> Repo.preload(:google_auth) query = Stats.Query.from(site.timezone, params) - search_terms = if site.google_auth && site.google_auth.property do + search_terms = if site.google_auth && site.google_auth.property && !query.filters["goal"] do Plausible.Google.Api.fetch_stats(site.google_auth, query) end diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 7422d1e61..b35daf793 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -37,19 +37,16 @@ defmodule Plausible.Stats.QueryTest do assert q.step_type == "month" end - test "defaults to 6 months format" do - assert Query.from(@tz, %{}) == Query.from(@tz, %{"period" => "6mo"}) + test "defaults to 30 days format" do + assert Query.from(@tz, %{}) == Query.from(@tz, %{"period" => "30d"}) end - test "parses custom format" do - q = Query.from(@tz, %{ - "period" => "custom", - "from" => "2019-01-01", - "to" => "2019-02-01" - }) + describe "filters" do + test "parses goal filter" do + filters = Jason.encode!(%{"goal" => "Signup"}) + q = Query.from(@tz, %{"period" => "3mo", "filters" => filters}) - assert q.date_range.first == ~D[2019-01-01] - assert q.date_range.last == ~D[2019-02-01] - assert q.step_type == "date" + assert q.filters["goal"] == "Signup" + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index 361645a21..d885c241f 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -5,11 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do describe "GET /api/stats/:domain/referrers" do setup [:create_user, :log_in, :create_site] - test "returns top referrer sources by new visitors", %{conn: conn, site: site} do - insert(:pageview, hostname: site.domain, referrer_source: "Google", new_visitor: true, timestamp: ~N[2019-01-01 01:00:00]) - insert(:pageview, hostname: site.domain, referrer_source: "Google", new_visitor: false, timestamp: ~N[2019-01-01 02:00:00]) - insert(:pageview, hostname: site.domain, referrer_source: "Google", new_visitor: true, timestamp: ~N[2019-01-01 02:00:00]) - insert(:pageview, hostname: site.domain, referrer_source: "Bing", new_visitor: true, timestamp: ~N[2019-01-01 02:00:00]) + test "returns top referrer sources by unique visitors", %{conn: conn, site: site} do + pageview1 = insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Google", user_id: pageview1.user_id, timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Bing", timestamp: ~N[2019-01-01 02:00:00]) conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01") From 5d820dea39eaec30b977daac6af84e7eb6bd3f9e Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:31:49 +0800 Subject: [PATCH 2/7] Add test for goal filter --- .../api/stats_controller/referrers_test.exs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index d885c241f..a233853b6 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -18,6 +18,19 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do %{"name" => "Bing", "count" => 1}, ] end + + test "filters referrers for a goal", %{conn: conn, site: site} do + insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) + insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + + filters = Jason.encode!(%{goal: "Signup"}) + conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Google", "count" => 2}, + ] + end end describe "GET /api/stats/:domain/referrer-drilldown" do From 7d55cdd2a25287d735a6ee5a5fd2d40e8e1b9f8f Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:32:00 +0800 Subject: [PATCH 3/7] Remove unused code in query --- lib/plausible/stats/query.ex | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 7ae9d28cb..aa84268b2 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,24 +1,6 @@ defmodule Plausible.Stats.Query do defstruct [date_range: nil, step_type: nil, period: nil, steps: nil, filters: %{}] - def new(attrs) do - attrs - |> Enum.into(%{}) - |> Map.put(:__struct__, __MODULE__) - end - - def month(date) do - %__MODULE__{ - date_range: Date.range(Timex.beginning_of_month(date), Timex.end_of_month(date)) - } - end - - def day(date) do - %__MODULE__{ - date_range: Date.range(date, date) - } - end - def shift_back(%__MODULE__{period: "day"} = query) do new_date = query.date_range.first |> Timex.shift(days: -1) Map.put(query, :date_range, Date.range(new_date, new_date)) From 0ada544fd6e1c26ea198004da4b3d3c3b6d5e1ee Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:32:31 +0800 Subject: [PATCH 4/7] Test query for 6 months --- test/plausible/stats/query_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index b35daf793..96dc2b42f 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -37,6 +37,14 @@ defmodule Plausible.Stats.QueryTest do assert q.step_type == "month" end + test "parses 6 month format" do + q = Query.from(@tz, %{"period" => "6mo"}) + + assert q.date_range.first == Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() + assert q.date_range.last == Timex.today() + assert q.step_type == "month" + end + test "defaults to 30 days format" do assert Query.from(@tz, %{}) == Query.from(@tz, %{"period" => "30d"}) end From 9feecf6741f574d11120e062facd207be2360ad8 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:33:47 +0800 Subject: [PATCH 5/7] Test pageview goal filter for referrers --- .../api/stats_controller/referrers_test.exs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index a233853b6..961bf4fe6 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -19,7 +19,7 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ] end - test "filters referrers for a goal", %{conn: conn, site: site} do + test "filters referrers for a custom goal", %{conn: conn, site: site} do insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) insert(:event, name: "Signup", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) insert(:pageview, hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) @@ -31,6 +31,19 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do %{"name" => "Google", "count" => 2}, ] end + + test "filters referrers for a pageview goal", %{conn: conn, site: site} do + insert(:pageview, pathname: "/register", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) + insert(:pageview, pathname: "/register", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) + insert(:pageview, pathname: "/irrelevant", hostname: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + + filters = Jason.encode!(%{goal: "Visit /register"}) + conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Google", "count" => 2}, + ] + end end describe "GET /api/stats/:domain/referrer-drilldown" do From f71f98a52106f228744c7f79c2c76d22397d81bf Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:42:45 +0800 Subject: [PATCH 6/7] Test Google referrer drilldown --- config/config.exs | 3 ++- config/test.exs | 3 ++- .../controllers/api/stats_controller.ex | 5 ++++- .../api/stats_controller/referrers_test.exs | 14 ++++++++++++++ test/support/google_api_mock.ex | 8 ++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 test/support/google_api_mock.ex diff --git a/config/config.exs b/config/config.exs index 8e2254dca..88d96281f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -42,7 +42,8 @@ config :ref_inspector, database_path: "priv/ref_inspector" config :plausible, - paddle_api: Plausible.Billing.PaddleApi + paddle_api: Plausible.Billing.PaddleApi, + google_api: Plausible.Google.Api # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 979fbe82e..1bbd3e5ca 100644 --- a/config/test.exs +++ b/config/test.exs @@ -24,4 +24,5 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter config :plausible, - paddle_api: Plausible.PaddleApi.Mock + paddle_api: Plausible.PaddleApi.Mock, + google_api: Plausible.Google.Api.Mock diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index af48b7d75..95312470e 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -32,12 +32,15 @@ defmodule PlausibleWeb.Api.StatsController do json(conn, Stats.top_referrers(site, query, params["limit"] || 5)) end + + @google_api Application.fetch_env!(:plausible, :google_api) + def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do site = conn.assigns[:site] |> Repo.preload(:google_auth) query = Stats.Query.from(site.timezone, params) search_terms = if site.google_auth && site.google_auth.property && !query.filters["goal"] do - Plausible.Google.Api.fetch_stats(site.google_auth, query) + @google_api.fetch_stats(site.google_auth, query) end case search_terms do diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index 961bf4fe6..38b657d93 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -84,5 +84,19 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do ] } end + + test "gets keywords from Google", %{conn: conn, user: user, site: site} do + insert(:google_auth, user: user, user: user,site: site, property: "sc-domain:example.com") + insert(:pageview, hostname: site.domain, referrer: "google.com", referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00]) + insert(:pageview, hostname: site.domain, referrer: "google.com", referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00]) + + conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day&date=2019-01-01") + {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil) + + assert json_response(conn, 200) == %{ + "total_visitors" => 2, + "search_terms" => terms + } + end end end diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.ex new file mode 100644 index 000000000..5060242a1 --- /dev/null +++ b/test/support/google_api_mock.ex @@ -0,0 +1,8 @@ +defmodule Plausible.Google.Api.Mock do + def fetch_stats(_auth, _query) do + {:ok, [ + %{"name" => "simple web analytics", "count" => 6}, + %{"name" => "open-source analytics", "count" => 2}, + ]} + end +end From b5797c8b10f4690217ff6ca9065ede42c1b1ba32 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 20 Nov 2019 16:49:22 +0800 Subject: [PATCH 7/7] Test getting conversions with a goal filter --- lib/plausible/stats/stats.ex | 6 +++--- .../api/stats_controller/conversions_test.exs | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/plausible/stats/stats.ex b/lib/plausible/stats/stats.ex index 2c81a2245..5105381ef 100644 --- a/lib/plausible/stats/stats.ex +++ b/lib/plausible/stats/stats.ex @@ -205,10 +205,10 @@ defmodule Plausible.Stats do def goal_conversions(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do Repo.all(from e in base_query(site, query), - select: %{name: ^goal, count: count(e.user_id, :distinct)}, + select: count(e.user_id, :distinct), group_by: e.name, - order_by: [desc: 2] - ) + order_by: [desc: 1] + ) |> Enum.map(fn count -> %{name: goal, count: count} end) end def goal_conversions(site, query) do diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 318d77621..6574ec111 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -54,4 +54,23 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end end + + describe "GET /api/stats/:domain/conversions - with goal filter" do + setup [:create_user, :log_in, :create_site] + + test "returns only the conversion tha is filtered for", %{conn: conn, site: site} do + insert(:goal, %{domain: site.domain, page_path: "/success"}) + insert(:goal, %{domain: site.domain, event_name: "Signup"}) + + insert(:event, name: "Signup", hostname: site.domain, timestamp: ~N[2019-01-01 01:00:00]) + insert(:event, name: "pageview", pathname: "/success", hostname: site.domain, timestamp: ~N[2019-01-01 01:00:00]) + + filters = Jason.encode!(%{goal: "Signup"}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Signup", "count" => 1}, + ] + end + end end