From 40275b64d40cdeca1ddde231b58172ddedf00745 Mon Sep 17 00:00:00 2001 From: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Date: Thu, 21 Apr 2022 11:47:15 +0300 Subject: [PATCH] Pageview custom dimensions (#1816) * added custom dimension filtering tests for pages * first filter UI in place * pages, entry pages and exit pages can be filtered by pageview props * added tests for expected filtering behaviour * fix dimension filter for sources + tests * added is_not filtering functionality * fixed formatting * fixed admin_test * added (none) as filter value + is_not filter type in UI * added prefilling applied filter values and some UI tweaks * added fetch options * Make prop suggestions work with `props` filter * Fix test * Track login state internally * Add CHANGELOG entry Co-authored-by: Uku Taht --- CHANGELOG.md | 1 + assets/js/dashboard/filters.js | 28 +- assets/js/dashboard/query.js | 4 +- assets/js/dashboard/stats/modals/filter.js | 54 ++- lib/plausible/session/clickhouse_schema.ex | 3 + lib/plausible/session/store.ex | 4 +- lib/plausible/stats/base.ex | 82 +++- lib/plausible/stats/breakdown.ex | 10 +- lib/plausible/stats/filter_suggestions.ex | 80 +++- lib/plausible/stats/filters.ex | 2 +- lib/plausible/stats/query.ex | 43 +- .../controllers/api/stats_controller.ex | 43 +- .../templates/layout/_tracking.html.eex | 18 +- ...20404123000_add_entry_props_to_session.exs | 9 + test/plausible/session/store_test.exs | 6 +- test/plausible/site/admin_test.exs | 2 +- .../CSVs/30d-filter-path/exit_pages.csv | 2 +- .../api/stats_controller/browsers_test.exs | 69 +++ .../api/stats_controller/conversions_test.exs | 163 +++++++ .../api/stats_controller/countries_test.exs | 157 +++++++ .../operating_systems_test.exs | 73 +++ .../api/stats_controller/pages_test.exs | 430 +++++++++++++++++- .../stats_controller/screen_sizes_test.exs | 69 +++ .../api/stats_controller/sources_test.exs | 285 ++++++++++++ .../api/stats_controller/suggestions_test.exs | 114 +++++ 25 files changed, 1638 insertions(+), 113 deletions(-) create mode 100644 priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc807de6b..1fa9303d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file. - The breakdown endpoint with the property query `property=event:goal` returns custom goal properties (within `props`) - Added IPv6 Ecto support (via the environment-variable `ECTO_IPV6`) - New filter type: `contains`, available for `page`, `entry_page`, `exit_page` +- Add filter for custom property ### Fixed - UI fix where multi-line text in pills would not be underlined properly on small screens. diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 1ad4373c8..441f08477 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -17,7 +17,6 @@ function removeFilter(key, history, query) { const newOpts = { [key]: false } - if (key === 'goal') { newOpts.props = false } if (key === 'country') { newOpts.country_name = false } if (key === 'region') { newOpts.region_name = false } if (key === 'city') { newOpts.city_name = false } @@ -86,19 +85,6 @@ function filterText(key, rawValue, query) { } function renderDropdownFilter(site, history, [key, value], query) { - if (key === 'props') { - return ( - -
- {filterText(key, value, query)} - removeFilter(key, history, query)}> - - -
-
- ) - } - return (
@@ -249,17 +235,9 @@ class Filters extends React.Component { renderListFilter(history, [key, value], query) { return ( - {key === 'props' ? ( - - {filterText(key, value, query)} - - ) : ( - <> - - {filterText(key, value, query)} - - - )} + + {filterText(key, value, query)} + removeFilter(key, history, query)}> diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index 9c8774796..34356b349 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -157,7 +157,9 @@ export function eventName(query) { export const formattedFilters = { 'goal': 'Goal', - 'props': 'Goal properties', + 'props': 'Property', + 'prop_key': 'Property', + 'prop_value': 'Value', 'source': 'Source', 'utm_medium': 'UTM Medium', 'utm_source': 'UTM Source', diff --git a/assets/js/dashboard/stats/modals/filter.js b/assets/js/dashboard/stats/modals/filter.js index 22acc68a9..9d38dc42a 100644 --- a/assets/js/dashboard/stats/modals/filter.js +++ b/assets/js/dashboard/stats/modals/filter.js @@ -18,10 +18,26 @@ export const FILTER_GROUPS = { 'browser': ['browser', 'browser_version'], 'os': ['os', 'os_version'], 'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'], - 'goal': ['goal'] + 'goal': ['goal'], + 'props': ['prop_key', 'prop_value'] } function getFormState(filterGroup, query) { + if (filterGroup === 'props') { + const propsObject = query.filters['props'] + const entries = propsObject && Object.entries(propsObject) + + if (entries && entries.length == 1) { + const propKey = entries[0][0] + const propValue = valueWithoutPrefix(entries[0][1]) + + return { + 'prop_key': {name: propKey, value: propKey, type: FILTER_TYPES.is}, + 'prop_value': {name: propValue, value: propValue, type: toFilterType(entries[0][1])} + } + } + } + return FILTER_GROUPS[filterGroup].reduce((result, filter) => { const rawFilterValue = query.filters[filter] || '' const type = toFilterType(rawFilterValue) @@ -74,6 +90,10 @@ function supportsContains(filterName) { return ['page', 'entry_page', 'exit_page'].includes(filterName) } +function supportsIsNot(filterName) { + return !['goal', 'prop_key'].includes(filterName) +} + function withIndefiniteArticle(word) { if (word.startsWith('UTM')) { return `a ${ word}` @@ -89,6 +109,8 @@ export function formatFilterGroup(filterGroup) { return 'UTM tags' } else if (filterGroup === 'location') { return 'Location' + } else if (filterGroup === 'props') { + return 'Property' } else { return formattedFilters[filterGroup] } @@ -143,6 +165,13 @@ class FilterModal extends React.Component { if (filterKey === 'country') { res.push({filter: 'country_name', value: name}) } if (filterKey === 'region') { res.push({filter: 'region_name', value: name}) } if (filterKey === 'city') { res.push({filter: 'city_name', value: name}) } + if (filterKey === 'prop_value') {return res} + if (filterKey === 'prop_key') { + let propValue = formState['prop_value'] + let filterValue = JSON.stringify({ [value]: toFilterQuery(propValue.value, propValue.type) }) + res.push({filter: 'props', value: filterValue}) + return res + } res.push({filter: filterKey, value: toFilterQuery(value, type)}) return res @@ -187,10 +216,20 @@ class FilterModal extends React.Component { const formFilters = Object.fromEntries( Object.entries(formState).map(([k, v]) => [k, v.code || v.value]) ) - const updatedQuery = {...query, filters: { ...query.filters, ...formFilters, [filter]: null }} - + const updatedQuery = this.queryForSuggestions(query, formFilters, filter) return api.get(apiPath(this.props.site, `/suggestions/${filter}`), updatedQuery, { q: input.trim() }) + } + } + queryForSuggestions(query, formFilters, filter) { + if (filter === 'prop_key') { + const propsFilter = formFilters.prop_value ? {'': formFilters.prop_value} : null + return {...query, filters: { ...query.filters, props: propsFilter}} + } else if (filter === 'prop_value') { + const propsFilter = formFilters.prop_key ? {[formFilters.prop_key]: '!(none)'} : null + return {...query, filters: { ...query.filters, props: propsFilter}} + } else { + return {...query, filters: { ...query.filters, ...formFilters, [filter]: null }} } } @@ -225,7 +264,6 @@ class FilterModal extends React.Component {
{ formattedFilters[filter] }
{ this.renderFilterTypeSelector(filter) } -
{ this.renderTypeItem(filterName, FILTER_TYPES.is, true) } - { this.renderTypeItem(filterName, FILTER_TYPES.isNot, filterName !== 'goal') } + { this.renderTypeItem(filterName, FILTER_TYPES.isNot, supportsIsNot(filterName)) } { this.renderTypeItem(filterName, FILTER_TYPES.contains, supportsContains(filterName)) }
@@ -299,9 +337,9 @@ class FilterModal extends React.Component { ); } - renderBody() { - const { selectedFilterGroup, query } = this.state; - const showClear = FILTER_GROUPS[selectedFilterGroup].some((filterName) => query.filters[filterName]) + renderBody() { + const { selectedFilterGroup, query } = this.state; + const showClear = FILTER_GROUPS[selectedFilterGroup].some((filterName) => query.filters[filterName]) return ( <> diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index 00b0b1f31..b66188a7b 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -18,6 +18,9 @@ defmodule Plausible.ClickhouseSession do field :events, :integer field :sign, :integer + field :"entry.meta.key", {:array, :string}, default: [] + field :"entry.meta.value", {:array, :string}, default: [] + field :utm_medium, :string field :utm_source, :string field :utm_campaign, :string diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index b515290e6..9c8cfbccd 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -160,7 +160,9 @@ defmodule Plausible.Session.Store do browser: event.browser, browser_version: event.browser_version, timestamp: event.timestamp, - start: event.timestamp + start: event.timestamp, + "entry.meta.key": Map.get(event, :"meta.key"), + "entry.meta.value": Map.get(event, :"meta.value") } end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 660edadc8..0f65815d5 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -87,29 +87,45 @@ defmodule Plausible.Stats.Base do q end - Enum.reduce(query.filters, q, fn {filter_key, filter_value}, query -> - case filter_key do - "event:props:" <> prop_name -> - filter_value = elem(filter_value, 1) - - if filter_value == "(none)" do + q = + case Query.get_filter_by_prefix(query, "event:props") do + {"event:props:" <> prop_name, {:is, value}} -> + if value == "(none)" do from( - e in query, + e in q, where: fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name) ) else from( - e in query, + e in q, inner_lateral_join: meta in "meta", as: :meta, - where: meta.key == ^prop_name and meta.value == ^filter_value + where: meta.key == ^prop_name and meta.value == ^value + ) + end + + {"event:props:" <> prop_name, {:is_not, value}} -> + if value == "(none)" do + from( + e in q, + where: fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) + ) + else + from( + e in q, + left_lateral_join: meta in "meta", + as: :meta, + where: + (meta.key == ^prop_name and meta.value != ^value) or + fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name) ) end _ -> - query + q end - end) + + q end @api_prop_name_to_db %{ @@ -158,6 +174,48 @@ defmodule Plausible.Stats.Base do ) end + sessions_q = + case Query.get_filter_by_prefix(query, "visit:entry_props:") do + nil -> + sessions_q + + {"visit:entry_props:" <> prop_name, filter_value} -> + case filter_value do + {:is, "(none)"} -> + from( + s in sessions_q, + where: fragment("not has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + ) + + {:is, value} -> + from( + s in sessions_q, + inner_lateral_join: meta in "entry.meta", + as: :meta, + where: meta.key == ^prop_name and meta.value == ^value + ) + + {:is_not, "(none)"} -> + from( + s in sessions_q, + where: fragment("has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + ) + + {:is_not, value} -> + from( + s in sessions_q, + left_lateral_join: meta in "entry.meta", + as: :meta, + where: + (meta.key == ^prop_name and meta.value != ^value) or + fragment("not has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + ) + + _ -> + sessions_q + end + end + Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q -> filter = query.filters["visit:" <> prop_name] @@ -307,7 +365,7 @@ defmodule Plausible.Stats.Base do end def filter_converted_sessions(db_query, site, query) do - if query.filters["event:name"] || query.filters["event:page"] || query.filters["event:goal"] do + if Query.has_event_filters?(query) do converted_sessions = from(e in query_events(site, query), select: %{session_id: fragment("DISTINCT ?", e.session_id)} diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index a8de13d89..654e0ccf8 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -146,7 +146,10 @@ defmodule Plausible.Stats.Breakdown do "visit:utm_content", "visit:utm_term" ] do - query = Query.treat_page_filter_as_entry_page(query) + query = + query + |> Query.treat_page_filter_as_entry_page() + |> Query.treat_prop_filter_as_entry_prop() breakdown_sessions(site, query, property, metrics, pagination) end @@ -223,10 +226,7 @@ defmodule Plausible.Stats.Breakdown do defp breakdown_time_on_page(site, query, pages) do q = from( - e in base_event_query(site, %Query{ - query - | filters: Map.delete(query.filters, "event:page") - }), + e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])), select: { fragment("? as p", e.pathname), fragment("? as t", e.timestamp), diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 569a3ceee..fdecb6718 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -2,6 +2,7 @@ defmodule Plausible.Stats.FilterSuggestions do use Plausible.Repo use Plausible.ClickhouseRepo import Plausible.Stats.Base + alias Plausible.Stats.Query def filter_suggestions(site, query, "country", filter_search) do matches = Location.search_country(filter_search) @@ -127,6 +128,54 @@ defmodule Plausible.Stats.FilterSuggestions do end) end + def filter_suggestions(site, query, "prop_key", filter_search) do + filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" + + q = + from(e in base_event_query(site, Query.remove_event_filters(query, [:props])), + inner_lateral_join: meta in "meta", + as: :meta, + select: meta.key, + where: fragment("? ilike ?", meta.key, ^filter_query), + group_by: meta.key, + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + + q = + case Query.get_filter_by_prefix(query, "event:props") do + {"event:props:", {:is, val}} -> from([e, m] in q, where: m.value == ^val) + _ -> q + end + + ClickhouseRepo.all(q) + |> Enum.filter(fn suggestion -> suggestion != "" end) + end + + def filter_suggestions(site, query, "prop_value", filter_search) do + filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" + + q = + from(e in base_event_query(site, query), + inner_lateral_join: meta in "meta", + as: :meta, + select: meta.value, + where: fragment("? ilike ?", meta.value, ^filter_query), + group_by: meta.value, + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + + q = + case Query.get_filter_by_prefix(query, "event:props") do + {"event:props:" <> key, _filter} -> from([e, m] in q, where: m.key == ^key) + _ -> q + end + + ClickhouseRepo.all(q) + |> Enum.filter(fn suggestion -> suggestion != "" end) + end + def filter_suggestions(site, query, filter_name, filter_search) do filter_search = if filter_search == nil, do: "", else: filter_search @@ -160,94 +209,93 @@ defmodule Plausible.Stats.FilterSuggestions do case filter_name do "pathname" -> from(e in q, - select: {e.pathname}, + select: e.pathname, where: fragment("? ilike ?", e.pathname, ^filter_query) ) "entry_page" -> from(e in q, - select: {e.entry_page}, + select: e.entry_page, where: fragment("? ilike ?", e.entry_page, ^filter_query) ) "exit_page" -> from(e in q, - select: {e.exit_page}, + select: e.exit_page, where: fragment("? ilike ?", e.exit_page, ^filter_query) ) "referrer_source" -> from(e in q, - select: {e.referrer_source}, + select: e.referrer_source, where: fragment("? ilike ?", e.referrer_source, ^filter_query) ) "utm_medium" -> from(e in q, - select: {e.utm_medium}, + select: e.utm_medium, where: fragment("? ilike ?", e.utm_medium, ^filter_query) ) "utm_source" -> from(e in q, - select: {e.utm_source}, + select: e.utm_source, where: fragment("? ilike ?", e.utm_source, ^filter_query) ) "utm_campaign" -> from(e in q, - select: {e.utm_campaign}, + select: e.utm_campaign, where: fragment("? ilike ?", e.utm_campaign, ^filter_query) ) "utm_content" -> from(e in q, - select: {e.utm_content}, + select: e.utm_content, where: fragment("? ilike ?", e.utm_content, ^filter_query) ) "utm_term" -> from(e in q, - select: {e.utm_term}, + select: e.utm_term, where: fragment("? ilike ?", e.utm_term, ^filter_query) ) "referrer" -> from(e in q, - select: {e.referrer}, + select: e.referrer, where: fragment("? ilike ?", e.referrer, ^filter_query) ) "browser" -> - from(e in q, select: {e.browser}, where: fragment("? ilike ?", e.browser, ^filter_query)) + from(e in q, select: e.browser, where: fragment("? ilike ?", e.browser, ^filter_query)) "browser_version" -> from(e in q, - select: {e.browser_version}, + select: e.browser_version, where: fragment("? ilike ?", e.browser_version, ^filter_query) ) "operating_system" -> from(e in q, - select: {e.operating_system}, + select: e.operating_system, where: fragment("? ilike ?", e.operating_system, ^filter_query) ) "operating_system_version" -> from(e in q, - select: {e.operating_system_version}, + select: e.operating_system_version, where: fragment("? ilike ?", e.operating_system_version, ^filter_query) ) "screen_size" -> from(e in q, - select: {e.screen_size}, + select: e.screen_size, where: fragment("? ilike ?", e.screen_size, ^filter_query) ) end ClickhouseRepo.all(q) - |> Enum.map(fn {suggestion} -> suggestion end) |> Enum.filter(fn suggestion -> suggestion != "" end) end end diff --git a/lib/plausible/stats/filters.ex b/lib/plausible/stats/filters.ex index 6bf09fe43..d65274f53 100644 --- a/lib/plausible/stats/filters.ex +++ b/lib/plausible/stats/filters.ex @@ -54,7 +54,7 @@ defmodule Plausible.Stats.Filters do name == "props" -> Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters -> - Map.put(new_filters, "event:props:" <> prop_key, {:is, prop_val}) + Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val)) end) end end) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 062a23ff9..ab12a28e4 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -272,19 +272,48 @@ defmodule Plausible.Stats.Query do def treat_page_filter_as_entry_page(q), do: q - def remove_goal(query) do - props = - Enum.map(query.filters, fn {key, _val} -> key end) - |> Enum.filter(fn filter_key -> String.starts_with?(filter_key, "event:props:") end) + def treat_prop_filter_as_entry_prop(%__MODULE__{filters: filters} = q) do + prop_filter = get_filter_by_prefix(q, "event:props:") + case {filters["event:goal"], prop_filter} do + {nil, {"event:props:" <> prop, filter_value}} -> + q + |> remove_event_filters([:props]) + |> put_filter("visit:entry_props:" <> prop, filter_value) + + _ -> + q + end + end + + def remove_event_filters(query, opts) do new_filters = - query.filters - |> Map.drop(props) - |> Map.delete("event:goal") + Enum.filter(query.filters, fn {filter_key, _} -> + cond do + :page in opts && filter_key == "event:page" -> false + :goal in opts && filter_key == "event:goal" -> false + :props in opts && filter_key && String.starts_with?(filter_key, "event:props:") -> false + true -> true + end + end) + |> Enum.into(%{}) %__MODULE__{query | filters: new_filters} end + def has_event_filters?(query) do + Enum.any?(query.filters, fn + {"event:" <> _, _} -> true + _ -> false + end) + end + + def get_filter_by_prefix(query, prefix) do + Enum.find(query.filters, fn {prop, _value} -> + String.starts_with?(prop, prefix) + end) + end + defp today(tz) do Timex.now(tz) |> Timex.to_date() end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 984620aa8..429854dd6 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -113,7 +113,7 @@ defmodule PlausibleWeb.Api.StatsController do end defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _goal}} = query) do - total_q = Query.remove_goal(query) + total_q = Query.remove_event_filters(query, [:goal, :props]) prev_query = Query.shift_back(query, site) prev_total_query = Query.shift_back(total_q, site) @@ -529,27 +529,28 @@ defmodule PlausibleWeb.Api.StatsController do total_visits_query = Query.put_filter(query, "event:page", {:member, pages}) - |> Query.put_filter("event:goal", nil) |> Query.put_filter("event:name", {:is, "pageview"}) - |> Query.put_filter("visit:goal", query.filters["event:goal"]) - |> Query.put_filter("visit:page", query.filters["event:page"]) - - total_pageviews = - Stats.breakdown(site, total_visits_query, "event:page", [:pageviews], {limit, 1}) exit_pages = - Enum.map(exit_pages, fn exit_page -> - exit_rate = - case Enum.find(total_pageviews, &(&1[:page] == exit_page[:name])) do - %{pageviews: pageviews} -> - Float.floor(exit_page[:total_exits] / pageviews * 100) - - nil -> - nil - end + if !Query.has_event_filters?(query) do + total_pageviews = + Stats.breakdown(site, total_visits_query, "event:page", [:pageviews], {limit, 1}) - Map.put(exit_page, :exit_rate, exit_rate) - end) + Enum.map(exit_pages, fn exit_page -> + exit_rate = + case Enum.find(total_pageviews, &(&1[:page] == exit_page[:name])) do + %{pageviews: pageviews} -> + Float.floor(exit_page[:total_exits] / pageviews * 100) + + nil -> + nil + end + + Map.put(exit_page, :exit_rate, exit_rate) + end) + else + exit_pages + end if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -821,7 +822,7 @@ defmodule PlausibleWeb.Api.StatsController do query end - total_q = Query.remove_goal(query) + total_q = Query.remove_event_filters(query, [:goal, :props]) %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) @@ -857,7 +858,7 @@ defmodule PlausibleWeb.Api.StatsController do query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) - total_q = Query.remove_goal(query) + total_q = Query.remove_event_filters(query, [:goal, :props]) %{:visitors => %{value: unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) @@ -985,7 +986,7 @@ defmodule PlausibleWeb.Api.StatsController do query_without_goal = query |> Query.put_filter(filter_name, {:member, items}) - |> Query.remove_goal() + |> Query.remove_event_filters([:goal, :props]) res_without_goal = Stats.breakdown(site, query_without_goal, filter_name, [:visitors], pagination) diff --git a/lib/plausible_web/templates/layout/_tracking.html.eex b/lib/plausible_web/templates/layout/_tracking.html.eex index e364ea969..6993d50e4 100644 --- a/lib/plausible_web/templates/layout/_tracking.html.eex +++ b/lib/plausible_web/templates/layout/_tracking.html.eex @@ -1,17 +1,17 @@ <%= if !Application.get_env(:plausible, :is_selfhost) && !@conn.assigns[:skip_plausible_tracking] do %> - <%= if Application.get_env(:plausible, :environment) == "prod" do %> - - - - <% end %> - - <%= if Application.get_env(:plausible, :environment) == "staging" do %> - + <%= if Application.get_env(:plausible, :environment) in ["prod", "staging"] do %> + + <% end %> <%= if Application.get_env(:plausible, :environment) == "dev" do %> - + + <% end %> <% end %> diff --git a/priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exs new file mode 100644 index 000000000..aa951a2d6 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20220404123000_add_entry_props_to_session.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :"entry.meta", {:nested, {{:key, :string}, {:value, :string}}} + end + end +end diff --git a/test/plausible/session/store_test.exs b/test/plausible/session/store_test.exs index fce9a7ca3..78eed21df 100644 --- a/test/plausible/session/store_test.exs +++ b/test/plausible/session/store_test.exs @@ -28,7 +28,9 @@ defmodule Plausible.Session.StoreTest do country_code: "EE", screen_size: "Desktop", operating_system: "Mac", - operating_system_version: "11" + operating_system_version: "11", + "meta.key": ["logged_in", "darkmode"], + "meta.value": ["true", "false"] ) Store.on_event(event, nil, store) @@ -59,6 +61,8 @@ defmodule Plausible.Session.StoreTest do assert session.browser_version == event.browser_version assert session.timestamp == event.timestamp assert session.start === event.timestamp + assert Map.get(session, :"entry.meta.key") == ["logged_in", "darkmode"] + assert Map.get(session, :"entry.meta.value") == ["true", "false"] end test "updates a session", %{store: store} do diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs index f5cd92538..1b2b6ed32 100644 --- a/test/plausible/site/admin_test.exs +++ b/test/plausible/site/admin_test.exs @@ -66,7 +66,7 @@ defmodule Plausible.SiteAdminTest do actual = SiteAdmin.session_transfer_query("from.com", "to.com") expected = - "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry.meta.key, entry.meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry.meta.key, entry.meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" assert actual == expected end diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv index 897181745..736ba6d58 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-path/exit_pages.csv @@ -1,2 +1,2 @@ name,unique_exits,total_exits,exit_rate -/some-other-page,1,1,100.0 +/some-other-page,1,1, diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 96fc77441..e55c734c1 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -20,6 +20,75 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do ] end + test "returns top browsers with :is filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + browser: "Chrome" + ), + build(:pageview, + user_id: 123, + browser: "Chrome", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + browser: "Safari" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Chrome", "visitors" => 1, "percentage" => 100} + ] + end + + test "returns top browsers with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + browser: "Chrome", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + user_id: 123, + browser: "Chrome", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + browser: "Safari" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Firefox", "visitors" => 1, "percentage" => 50}, + %{"name" => "Safari", "visitors" => 1, "percentage" => 50} + ] + end + test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, user_id: 1, browser: "Chrome"), diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index 3c189af19..8f41f0837 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -49,6 +49,169 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do } ] end + + test "returns conversions when a direct :is filter on event prop", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "true"] + ), + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["500", "true"] + ), + build(:event, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "false"] + ), + build(:event, + name: "Payment", + "meta.key": ["amount"], + "meta.value": ["200"] + ) + ]) + + insert(:goal, %{domain: site.domain, event_name: "Payment"}) + + filters = Jason.encode!(%{props: %{"logged_in" => "true"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Payment", + "unique_conversions" => 1, + "total_conversions" => 2, + "prop_names" => nil, + "conversion_rate" => 33.3 + } + ] + end + + test "returns conversions when a direct :is_not filter on event prop", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "true"] + ), + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["500", "true"] + ), + build(:event, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "false"] + ), + build(:event, name: "Payment") + ]) + + insert(:goal, %{domain: site.domain, event_name: "Payment"}) + + filters = Jason.encode!(%{props: %{"logged_in" => "!true"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Payment", + "unique_conversions" => 2, + "total_conversions" => 2, + "prop_names" => nil, + "conversion_rate" => 66.7 + } + ] + end + + test "returns conversions when a direct :is (none) filter on event prop", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + user_id: @user_id, + name: "Payment" + ), + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount"], + "meta.value": ["500"] + ), + build(:event, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "false"] + ), + build(:event, name: "Payment") + ]) + + insert(:goal, %{domain: site.domain, event_name: "Payment"}) + + filters = Jason.encode!(%{props: %{"logged_in" => "(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Payment", + "unique_conversions" => 2, + "total_conversions" => 3, + "prop_names" => nil, + "conversion_rate" => 66.7 + } + ] + end + + test "returns conversions when a direct :is_not (none) filter on event prop", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["500", "false"] + ), + build(:event, + user_id: @user_id, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["500", "true"] + ), + build(:event, + name: "Payment", + "meta.key": ["amount", "logged_in"], + "meta.value": ["100", "false"] + ), + build(:event, name: "Payment") + ]) + + insert(:goal, %{domain: site.domain, event_name: "Payment"}) + + filters = Jason.encode!(%{props: %{"logged_in" => "!(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Payment", + "unique_conversions" => 2, + "total_conversions" => 3, + "prop_names" => nil, + "conversion_rate" => 66.7 + } + ] + end end describe "GET /api/stats/:domain/conversions - with goal filter" do diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index dd0485b59..b727e7ceb 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -110,5 +110,162 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do } ] end + + test "returns top countries with :is filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + country_code: "EE" + ), + build(:pageview, + user_id: 123, + country_code: "EE", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + country_code: "GB", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + country_code: "US" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "code" => "EE", + "alpha_3" => "EST", + "name" => "Estonia", + "flag" => "🇪🇪", + "visitors" => 1, + "percentage" => 100 + } + ] + end + + test "returns top countries with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + country_code: "EE", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + user_id: 123, + country_code: "EE", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + country_code: "GB", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + country_code: "GB" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "code" => "GB", + "alpha_3" => "GBR", + "name" => "United Kingdom", + "flag" => "🇬🇧", + "visitors" => 2, + "percentage" => 100 + } + ] + end + + test "returns top countries with :is (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + country_code: "EE", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + country_code: "GB", + "meta.key": ["logged_in"], + "meta.value": ["true"] + ), + build(:pageview, + country_code: "GB" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "code" => "GB", + "alpha_3" => "GBR", + "name" => "United Kingdom", + "flag" => "🇬🇧", + "visitors" => 2, + "percentage" => 100 + } + ] + end + + test "returns top countries with :is_not (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + country_code: "EE", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + country_code: "EE", + "meta.key": ["author"], + "meta.value": [""] + ), + build(:pageview, + country_code: "GB", + "meta.key": ["logged_in"], + "meta.value": ["true"] + ), + build(:pageview, + country_code: "GB" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "code" => "EE", + "alpha_3" => "EST", + "name" => "Estonia", + "flag" => "🇪🇪", + "visitors" => 2, + "percentage" => 100 + } + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index f0d025ca8..ef8ee5467 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -42,6 +42,79 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do ] end + test "returns operating systems with :is filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + operating_system: "Mac" + ), + build(:pageview, + user_id: 123, + operating_system: "Mac", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + operating_system: "Windows", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + operating_system: "Android" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Mac", "visitors" => 1, "percentage" => 100} + ] + end + + test "returns screen sizes with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + operating_system: "Windows", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + user_id: 123, + operating_system: "Windows", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + operating_system: "Mac", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + operating_system: "Android" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + + conn = + get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Android", "visitors" => 1, "percentage" => 50}, + %{"name" => "Mac", "visitors" => 1, "percentage" => 50} + ] + end + test "returns operating systems by unique visitors with imported data", %{ conn: conn, site: site diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 9cacb00ce..5e0d02159 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -25,6 +25,335 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top pages with :is filter on custom pageview props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, user_id: 123, pathname: "/") + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"visitors" => 1, "name" => "/blog/john-1"} + ] + end + + test "returns top pages with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, pathname: "/") + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"visitors" => 1, "name" => "/"}, + %{"visitors" => 1, "name" => "/blog/other-post"} + ] + end + + test "calculates bounce_rate and time_on_page with :is filter on custom pageview props", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/john-2", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: @user_id, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:pageview, + pathname: "/blog/john-2", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 456, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog", + user_id: 456, + timestamp: ~N[2021-01-01 00:10:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/blog/john-2", + "visitors" => 2, + "pageviews" => 2, + "bounce_rate" => 0, + "time_on_page" => 600 + }, + %{ + "name" => "/blog/john-1", + "visitors" => 1, + "pageviews" => 1, + "bounce_rate" => 0, + "time_on_page" => 60 + } + ] + end + + test "calculates bounce_rate and time_on_page with :is_not filter on custom pageview props", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog/john-1", + user_id: @user_id, + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/other-post", + user_id: @user_id, + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:pageview, + pathname: "/blog", + user_id: 456, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 456, + timestamp: ~N[2021-01-01 00:03:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/blog", + "visitors" => 2, + "pageviews" => 2, + "bounce_rate" => 0, + "time_on_page" => 120.0 + }, + %{ + "name" => "/blog/other-post", + "visitors" => 1, + "pageviews" => 1, + "bounce_rate" => nil, + "time_on_page" => nil + } + ] + end + + test "calculates bounce_rate and time_on_page with :is (none) filter on custom pageview props", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog/john-1", + user_id: @user_id, + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/other-post", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:pageview, + pathname: "/blog", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/blog", + "visitors" => 2, + "pageviews" => 2, + "bounce_rate" => 50, + "time_on_page" => 60 + }, + %{ + "name" => "/blog/other-post", + "visitors" => 1, + "pageviews" => 1, + "bounce_rate" => nil, + "time_on_page" => nil + } + ] + end + + test "calculates bounce_rate and time_on_page with :is_not (none) filter on custom pageview props", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog/john-1", + user_id: @user_id, + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": ["other"], + user_id: @user_id, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": [""], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/blog/other-post", + "visitors" => 2, + "pageviews" => 2, + "bounce_rate" => 100, + "time_on_page" => nil + }, + %{ + "name" => "/blog/john-1", + "visitors" => 1, + "pageviews" => 1, + "bounce_rate" => 0, + "time_on_page" => 60 + } + ] + end + + test "calculates bounce_rate and time_on_page for pages filtered by page path", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/about", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:02:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/about", + timestamp: ~N[2021-01-01 00:10:00] + ) + ]) + + filters = Jason.encode!(%{page: "/"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/", + "visitors" => 2, + "pageviews" => 3, + "bounce_rate" => 50, + "time_on_page" => 60 + } + ] + end + test "returns top pages by visitors with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/"), @@ -241,6 +570,62 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top entry pages filtered by custom pageview props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog", + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/john-2", + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/blog", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "unique_entrances" => 1, + "total_entrances" => 1, + "name" => "/blog", + "visit_duration" => 60 + }, + %{ + "unique_entrances" => 1, + "total_entrances" => 1, + "name" => "/blog/john-2", + "visit_duration" => 0 + } + ] + end + test "returns top entry pages by visitors with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -416,6 +801,45 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top exit pages filtered by custom pageview props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/blog/john-1", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/", + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + pathname: "/blog/other-post", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "/", "unique_exits" => 1, "total_exits" => 1} + ] + end + test "returns top exit pages by visitors with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -523,7 +947,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "unique_exits" => 1, "total_visitors" => 1, "total_exits" => 1, - "exit_rate" => 50, "conversion_rate" => 100.0 }, %{ @@ -531,7 +954,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "unique_exits" => 1, "total_visitors" => 1, "total_exits" => 1, - "exit_rate" => 100, "conversion_rate" => 100.0 } ] @@ -575,8 +997,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ) assert json_response(conn, 200) == [ - %{"name" => "/exit1", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 50}, - %{"name" => "/exit2", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 100} + %{"name" => "/exit1", "unique_exits" => 1, "total_exits" => 1}, + %{"name" => "/exit2", "unique_exits" => 1, "total_exits" => 1} ] end end diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 029546cac..6582027d7 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -20,6 +20,75 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do ] end + test "returns screen sizes with :is filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + screen_size: "Desktop" + ), + build(:pageview, + user_id: 123, + screen_size: "Desktop", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + screen_size: "Mobile", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + screen_size: "Tablet" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Desktop", "visitors" => 1, "percentage" => 100} + ] + end + + test "returns screen sizes with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + user_id: 123, + screen_size: "Desktop", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + user_id: 123, + screen_size: "Desktop", + "meta.key": ["author"], + "meta.value": ["John Doe"] + ), + build(:pageview, + screen_size: "Mobile", + "meta.key": ["author"], + "meta.value": ["other"] + ), + build(:pageview, + screen_size: "Tablet" + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"name" => "Mobile", "visitors" => 1, "percentage" => 50}, + %{"name" => "Tablet", "visitors" => 1, "percentage" => 50} + ] + end + test "returns screen sizes by new visitors with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, screen_size: "Desktop"), diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 97f88ff59..0ece9bca9 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -30,6 +30,184 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end + test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 1} + ] + end + + test "returns top sources with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["other"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Facebook", "visitors" => 1}, + %{"name" => "Google", "visitors" => 1} + ] + end + + test "returns top sources with :is (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Facebook", "visitors" => 1} + ] + end + + test "returns top sources with :is_not (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["logged_in"], + "meta.value": ["true"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["other"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 1} + ] + end + test "returns top sources with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, referrer_source: "Google", referrer: "google.com"), @@ -532,6 +710,113 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end + test "returns top referrers with goal filter + :is prop filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Download", + "meta.key": ["method", "logged_in"], + "meta.value": ["HTTP", "true"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["logged_in"], + "meta.value": ["true"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + referrer_source: "Facebook", + referrer: "facebook.com", + name: "Download", + "meta.key": ["method", "logged_in"], + "meta.value": ["HTTP", "false"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{goal: "Download", props: %{"logged_in" => "true"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "DuckDuckGo", + "visitors" => 1, + "conversion_rate" => 50.0, + "total_visitors" => 2 + } + ] + end + + test "returns top referrers with goal filter + prop :is_not filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Download", + "meta.key": ["method"], + "meta.value": ["HTTP"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:event, + name: "Download", + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["method", "logged_in"], + "meta.value": ["HTTP", "true"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Download", + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["method", "logged_in"], + "meta.value": ["HTTP", "false"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{goal: "Download", props: %{"logged_in" => "!true"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Google", + "visitors" => 1, + "conversion_rate" => 50.0, + "total_visitors" => 2 + }, + %{ + "name" => "DuckDuckGo", + "visitors" => 1, + "conversion_rate" => 100.0, + "total_visitors" => 1 + } + ] + end + test "returns top referrers for a pageview goal including conversion_rate", %{ conn: conn, site: site diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs index d1bc1a270..f040008ed 100644 --- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs @@ -158,4 +158,118 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do assert json_response(conn, 200) == ["10words.com/page1"] end end + + describe "suggestions for props" do + setup [:create_user, :log_in, :create_new_site] + + test "returns suggestions for prop key with no filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Uku Taht"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["logged_in"], + "meta.value": ["false"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["dark_mode"], + "meta.value": ["true"], + timestamp: ~N[2022-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01") + + assert json_response(conn, 200) |> Enum.sort() == ["author", "dark_mode", "logged_in"] + end + + test "returns suggestions for prop key with value filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Uku Taht"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["logged_in"], + "meta.value": ["true"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["dark_mode"], + "meta.value": ["true"], + timestamp: ~N[2022-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"": "true"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) |> Enum.sort() == ["dark_mode", "logged_in"] + end + + test "returns suggestions for prop value with no filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Uku Taht"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["logged_in"], + "meta.value": ["false"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["dark_mode"], + "meta.value": ["true"], + timestamp: ~N[2022-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01") + + assert json_response(conn, 200) |> Enum.sort() == ["Uku Taht", "false", "true"] + end + + test "returns suggestions for prop value with filter on prop key", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Uku Taht"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["logged_in"], + "meta.value": ["false"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["dark_mode"], + "meta.value": ["true"], + timestamp: ~N[2022-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{author: "!(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) |> Enum.sort() == ["Uku Taht"] + end + end end