From caeff41932a47dc0e9ec124e12dd8229e8201bcf Mon Sep 17 00:00:00 2001 From: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Date: Thu, 27 Apr 2023 14:09:33 +0300 Subject: [PATCH] Prop filter modal (#2841) * Add PropFilterModal (only UI) * small variable refactor * allow selecting prop value filter type * allow selecting only one prop_key * allow selecting many prop_values only when prop_key selected * handle submitting filter * get applied filters from query + remove option * change prop filter label format * support member and not_member filter types for pageview props * show (none) value in filter suggestions * refactor zip_results/4 and remove unused code * fix displaying (none) values in goals section prop breakdown * remove unnecessary functionality * fix bug: returning prop names for goal :member filter * fix bug: submitting regular filter modal with Enter key * bugfix: disallow opening prop filter modal when feature flag disabled * mix format * break selected combobox values into multiple lines * fix useEffect behavior for focusing on prop_key field * support submitting prop filter with Enter key * refactor getFormState in PropFilterModal * separate fetchPropKey and fetchPropValue functions * Allow querying props for pageview goals * Make the internal props API only return a list of props (not map) * Separate function for fetching all props in Stats API goal breakdown (this returns a map as before) * ditch state for keeping search bar visible * group by event_name in db query --- assets/js/dashboard/components/combobox.js | 38 +- .../components/filter-type-selector.js | 70 ++- assets/js/dashboard/filters.js | 48 +- .../js/dashboard/stats/modals/filter-modal.js | 8 +- .../stats/modals/prop-filter-modal.js | 170 ++++++ .../stats/modals/regular-filter-modal.js | 50 +- assets/js/dashboard/util/filters.js | 4 +- lib/plausible/stats/base.ex | 28 + lib/plausible/stats/breakdown.ex | 45 +- lib/plausible/stats/custom_props.ex | 51 ++ lib/plausible/stats/filter_suggestions.ex | 32 +- lib/plausible/stats/mod.ex | 1 - lib/plausible/stats/props.ex | 34 -- .../api/external_stats_controller.ex | 4 +- .../controllers/api/stats_controller.ex | 21 +- .../breakdown_test.exs | 136 +++-- .../api/stats_controller/conversions_test.exs | 555 +++++++++++++++++- .../api/stats_controller/pages_test.exs | 88 +++ .../api/stats_controller/suggestions_test.exs | 93 ++- 19 files changed, 1180 insertions(+), 296 deletions(-) create mode 100644 assets/js/dashboard/stats/modals/prop-filter-modal.js create mode 100644 lib/plausible/stats/custom_props.ex diff --git a/assets/js/dashboard/components/combobox.js b/assets/js/dashboard/components/combobox.js index 7a727d422..fc5e99bc4 100644 --- a/assets/js/dashboard/components/combobox.js +++ b/assets/js/dashboard/components/combobox.js @@ -129,17 +129,24 @@ export default function PlausibleCombobox(props) { function selectOption(option) { if (isDisabled(option)) return - props.onSelect([...props.values, option]) + if (props.singleOption) { + props.onSelect([option]) + } else { + searchRef.current.focus() + props.onSelect([...props.values, option]) + } + setOpen(false) setInput('') - searchRef.current.focus() } function removeOption(option, e) { e.stopPropagation() const newValues = props.values.filter((val) => val.value !== option.value) props.onSelect(newValues) - searchRef.current.focus() + if (!searchBoxHidden) { + searchRef.current.focus() + } setOpen(false) } @@ -155,19 +162,38 @@ export default function PlausibleCombobox(props) { return () => { document.removeEventListener("mousedown", handleClick, false); } }, []) + useEffect(() => { + if (props.singleOption && props.values.length === 0) { + searchRef.current.focus() + } + }, [props.values.length === 0]) + const matchesFound = !loading && visibleOptions.length > 0 const noMatchesFound = !loading && visibleOptions.length === 0 + + const searchBoxHidden = !!props.singleOption && props.values.length === 1 + const searchBoxClass = classNames('border-none py-1 px-1 p-0 w-full inline-block rounded-md focus:outline-none focus:ring-0 text-sm', { + 'hidden': searchBoxHidden + }) + + const containerClass = classNames('relative w-full', { + [props.className]: !!props.className, + 'opacity-20 cursor-default pointer-events-none': props.isDisabled + }) return ( -
+
{ props.values.map((value) => { return ( -
{value.label} removeOption(value, e)} className="cursor-pointer font-bold ml-1">×
+
+ {value.label} + removeOption(value, e)} className="cursor-pointer font-bold ml-1">× +
) }) } - +
{!loading && } {loading && } diff --git a/assets/js/dashboard/components/filter-type-selector.js b/assets/js/dashboard/components/filter-type-selector.js index 5fbaa2da8..ac343bd1a 100644 --- a/assets/js/dashboard/components/filter-type-selector.js +++ b/assets/js/dashboard/components/filter-type-selector.js @@ -30,40 +30,46 @@ export default function FilterTypeSelector(props) { ) } + const containerClass = classNames("w-full", { + 'opacity-20 cursor-default pointer-events-none': props.isDisabled + }) + return ( - - {({ open }) => ( - <> -
- - {props.selectedType} - -
+
+ + {({ open }) => ( + <> +
+ + {props.selectedType} + +
- - -
- {renderTypeItem(FILTER_TYPES.is, true)} - {renderTypeItem(FILTER_TYPES.isNot, supportsIsNot(filterName))} - {renderTypeItem(FILTER_TYPES.contains, isFreeChoiceFilter(filterName))} -
-
-
- - )} -
+ +
+ {renderTypeItem(FILTER_TYPES.is, true)} + {renderTypeItem(FILTER_TYPES.isNot, supportsIsNot(filterName))} + {renderTypeItem(FILTER_TYPES.contains, isFreeChoiceFilter(filterName))} +
+
+ + + )} +
+
) } \ No newline at end of file diff --git a/assets/js/dashboard/filters.js b/assets/js/dashboard/filters.js index 2d2e63ebd..97f98bc4b 100644 --- a/assets/js/dashboard/filters.js +++ b/assets/js/dashboard/filters.js @@ -43,7 +43,7 @@ function filterText(key, _rawValue, query) { if (key === "props") { const [[propKey, _propValue]] = Object.entries(query.filters['props']) - return <>props.{propKey} {type} {clauses.map(({label}) => {label}).reduce((prev, curr) => [prev, ' or ', curr])} + return <>Property {propKey} {type} {clauses.map(({label}) => {label}).reduce((prev, curr) => [prev, ' or ', curr])} } else if (formattedFilter) { return <>{formattedFilter} {type} {clauses.map(({label}) => {label}).reduce((prev, curr) => [prev, ' or ', curr])} } @@ -55,15 +55,26 @@ function renderDropdownFilter(site, history, [key, value], query) { return (
- - {filterText(key, value, query)} - - + {shouldLinkToFilterModal(site, key) && + + {filterText(key, value, query)} + + + } + {!shouldLinkToFilterModal(site, key) && +
{e.preventDefault()}} + className="group flex w-full justify-between items-center" + style={{ width: 'calc(100% - 1.5rem)' }} + > + {filterText(key, value, query)} +
+ } removeFilter(key, history, query)}> @@ -72,6 +83,10 @@ function renderDropdownFilter(site, history, [key, value], query) { ) } +function shouldLinkToFilterModal(site, key) { + return key !== 'props' || site.flags.custom_dimension_filter +} + function filterDropdownOption(site, option) { return ( @@ -204,9 +219,16 @@ class Filters extends React.Component { renderListFilter(history, [key, value], query) { return ( - - {filterText(key, value, query)} - + {shouldLinkToFilterModal(this.props.site, key) && + + {filterText(key, value, query)} + + } + {!shouldLinkToFilterModal(this.props.site, key) && +
+ {filterText(key, value, query)} +
+ } removeFilter(key, history, query)}> diff --git a/assets/js/dashboard/stats/modals/filter-modal.js b/assets/js/dashboard/stats/modals/filter-modal.js index f8accc2ec..c6fbb3966 100644 --- a/assets/js/dashboard/stats/modals/filter-modal.js +++ b/assets/js/dashboard/stats/modals/filter-modal.js @@ -2,12 +2,16 @@ import React from "react" import { withRouter } from 'react-router-dom' import Modal from './modal' import RegularFilterModal from './regular-filter-modal' +import PropFilterModal from "./prop-filter-modal" function FilterModal(props) { function renderBody() { const filterGroup = props.match.params.field || 'page' - - return + if (filterGroup === 'props') { + return + } else { + return + } } return ( diff --git a/assets/js/dashboard/stats/modals/prop-filter-modal.js b/assets/js/dashboard/stats/modals/prop-filter-modal.js new file mode 100644 index 000000000..9c8e6b49e --- /dev/null +++ b/assets/js/dashboard/stats/modals/prop-filter-modal.js @@ -0,0 +1,170 @@ +import React, { useEffect, useState } from 'react' +import { withRouter } from "react-router-dom"; + +import Combobox from '../../components/combobox' +import FilterTypeSelector from "../../components/filter-type-selector"; +import { FILTER_TYPES } from "../../util/filters"; +import { parseQuery } from '../../query' +import * as api from '../../api' +import { apiPath, siteBasePath } from '../../util/url' +import { toFilterQuery, parseQueryFilter } from '../../util/filters'; +import { shouldIgnoreKeypress } from '../../keybinding'; + +function getFormState(query) { + const rawValue = query.filters['props'] + if (rawValue) { + const [[propKey, _propValue]] = Object.entries(rawValue) + const {type, clauses} = parseQueryFilter(query, 'props') + + return { + prop_key: {value: propKey, label: propKey}, + prop_value: { type: type, clauses: clauses } + } + } + + return { + prop_key: null, + prop_value: { type: FILTER_TYPES.is, clauses: [] } + } +} + +function PropFilterModal(props) { + const query = parseQuery(props.location.search, props.site) + const [formState, setFormState] = useState(getFormState(query)) + + function fetchPropKeyOptions() { + return (input) => { + return api.get(apiPath(props.site, "/suggestions/prop_key"), query, { q: input.trim() }) + } + } + + function fetchPropValueOptions() { + return (input) => { + const propKey = formState.prop_key?.value + const updatedQuery = { ...query, filters: { ...query.filters, props: {[propKey]: '!(none)'} } } + return api.get(apiPath(props.site, "/suggestions/prop_value"), updatedQuery, { q: input.trim() }) + } + } + + function onPropKeySelect() { + return (selectedOptions) => { + const newPropKey = selectedOptions.length === 0 ? null : selectedOptions[0] + setFormState(prevState => ({ + prop_key: newPropKey, + prop_value: { type: prevState.prop_value.type, clauses: [] } + })) + } + } + + function onPropValueSelect() { + return (selection) => { + setFormState(prevState => ({ + ...prevState, prop_value: { ...prevState.prop_value, clauses: selection } + })) + } + } + + function onFilterTypeSelect() { + return (newType) => { + setFormState(prevState => ({ + ...prevState, prop_value: { ...prevState.prop_value, type: newType } + })) + } + } + + function selectedFilterType() { + return formState.prop_value.type + } + + function renderFilterInputs() { + return ( +
+
+ +
+
+ +
+
+ +
+
+ ) + } + + function isDisabled() { + return !(formState.prop_key && formState.prop_value.clauses.length > 0) + } + + function shouldShowClear() { + return !!query.filters['props'] + } + + function handleSubmit() { + const filterString = JSON.stringify({ [formState.prop_key.value]: toFilterQuery(formState.prop_value.type, formState.prop_value.clauses) }) + selectFiltersAndCloseModal(filterString) + } + + function selectFiltersAndCloseModal(filterString) { + const queryString = new URLSearchParams(window.location.search) + + if (filterString) { + queryString.set('props', filterString) + } else { + queryString.delete('props') + } + + props.history.replace({ pathname: siteBasePath(props.site), search: queryString.toString() }) + } + + const handleKeydown = (e) => { + if (shouldIgnoreKeypress(e)) return + + if (e.target.tagName === 'BODY' && e.key === 'Enter') { + handleSubmit() + } + } + + useEffect(() => { + document.addEventListener('keydown', handleKeydown) + return () => { + document.removeEventListener('keydown', handleKeydown) + } + }, [handleSubmit]) + + return ( + <> +

Filter by Property

+ +
+
+
+ {renderFilterInputs()} + +
+ + + {shouldShowClear() && ( + + )} +
+
+
+ + ) +} + +export default withRouter(PropFilterModal) \ No newline at end of file diff --git a/assets/js/dashboard/stats/modals/regular-filter-modal.js b/assets/js/dashboard/stats/modals/regular-filter-modal.js index 9a5dd5ed6..a3fdcfc17 100644 --- a/assets/js/dashboard/stats/modals/regular-filter-modal.js +++ b/assets/js/dashboard/stats/modals/regular-filter-modal.js @@ -11,21 +11,6 @@ import { shouldIgnoreKeypress } from '../../keybinding' import { isFreeChoiceFilter } from "../../util/filters"; function getFormState(filterGroup, query) { - if (filterGroup === 'props') { - const propsObject = query.filters['props'] - const entries = propsObject && Object.entries(propsObject) - - if (entries && entries.length == 1) { - const [[propKey, _propVal]] = entries - const {type, clauses} = parseQueryFilter(query, 'props') - - return { - 'prop_key': { type: FILTER_TYPES.is, clauses: [{label: propKey, value: propKey}] }, - 'prop_value': { type, clauses } - } - } - } - return FILTER_GROUPS[filterGroup].reduce((result, filter) => { const {type, clauses} = parseQueryFilter(query, filter) @@ -47,7 +32,8 @@ class RegularFilterModal extends React.Component { super(props) const query = parseQuery(props.location.search, props.site) const formState = getFormState(props.filterGroup, query) - + + this.handleKeydown = this.handleKeydown.bind(this) this.state = { query, formState } } @@ -75,12 +61,6 @@ class RegularFilterModal extends React.Component { if (filterKey === 'country') { res.push({ filter: 'country_labels', value: clauses.map(clause => clause.label).join('|') }) } if (filterKey === 'region') { res.push({ filter: 'region_labels', value: clauses.map(clause => clause.label).join('|') }) } if (filterKey === 'city') { res.push({ filter: 'city_labels', value: clauses.map(clause => clause.label).join('|') }) } - if (filterKey === 'prop_value') { return res } - if (filterKey === 'prop_key') { - const [{value: propKey}] = clauses - res.push({ filter: 'props', value: JSON.stringify({ [propKey]: toFilterQuery(formState.prop_value.type, formState.prop_value.clauses) }) }) - return res - } res.push({ filter: filterKey, value: toFilterQuery(type, clauses) }) return res @@ -125,15 +105,7 @@ class RegularFilterModal extends React.Component { } queryForSuggestions(query, formFilters, filter) { - if (filter === 'prop_key') { - const propsFilter = formFilters.prop_value ? { '': formFilters.prop_value } : null - return { ...query, filters: { ...query.filters, props: propsFilter } } - } else if (filter === 'prop_value') { - const propsFilter = formFilters.prop_key ? { [formFilters.prop_key]: '!(none)' } : null - return { ...query, filters: { ...query.filters, props: propsFilter } } - } else { - return { ...query, filters: { ...query.filters, ...formFilters, [filter]: this.negate(formFilters[filter]) } } - } + return { ...query, filters: { ...query.filters, ...formFilters, [filter]: this.negate(formFilters[filter]) } } } negate(filterVal) { @@ -153,11 +125,7 @@ class RegularFilterModal extends React.Component { } isDisabled() { - if (this.props.filterGroup === 'props') { - return Object.entries(this.state.formState).some(([_key, { clauses }]) => clauses.length === 0) - } else { - return Object.entries(this.state.formState).every(([_key, { clauses }]) => clauses.length === 0) - } + return Object.entries(this.state.formState).every(([_key, { clauses }]) => clauses.length === 0) } selectFiltersAndCloseModal(filters) { @@ -181,9 +149,13 @@ class RegularFilterModal extends React.Component { return (
{formattedFilters[filter]}
-
- - +
+
+ +
+
+ +
) diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.js index baed92371..b0da22b32 100644 --- a/assets/js/dashboard/util/filters.js +++ b/assets/js/dashboard/util/filters.js @@ -55,9 +55,7 @@ function parsePrefix(rawValue) { const type = Object.keys(FILTER_PREFIXES) .find(type => FILTER_PREFIXES[type] === rawValue[0]) || FILTER_TYPES.is; - const value = [FILTER_TYPES.isNot, FILTER_TYPES.contains].includes(type) - ? rawValue.substring(1) - : rawValue; + const value = type === FILTER_TYPES.is ? rawValue : rawValue.substring(1) const values = value .split(NON_ESCAPED_PIPE_REGEX) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index d93624ef6..8fa3dc524 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -167,6 +167,34 @@ defmodule Plausible.Stats.Base do ) end + {"event:props:" <> prop_name, {:member, values}} -> + none_value_included = Enum.member?(values, "(none)") + + from( + e in q, + left_lateral_join: meta in "meta", + as: :meta, + where: + (meta.key == ^prop_name and meta.value in ^values) or + (^none_value_included and + fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)) + ) + + {"event:props:" <> prop_name, {:not_member, values}} -> + none_value_included = Enum.member?(values, "(none)") + + from( + e in q, + left_lateral_join: meta in "meta", + as: :meta, + where: + (meta.key == ^prop_name and meta.value not in ^values) or + (^none_value_included and fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) and + meta.value not in ^values) or + (not (^none_value_included) and + fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)) + ) + _ -> q end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index ab52a8071..91db6b8b5 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -68,12 +68,10 @@ defmodule Plausible.Stats.Breakdown do {limit, _} = pagination none_result = - if !Enum.any?(query.filters, fn {key, _} -> String.starts_with?(key, "event:props") end) do - none_filters = Map.put(query.filters, "event:props:" <> custom_prop, {:is, "(none)"}) - none_query = %Query{query | filters: none_filters} + if include_none_result?(query.filters[property]) do + none_query = Query.put_filter(query, property, {:is, "(none)"}) from(e in base_event_query(site, none_query), - order_by: [desc: fragment("uniq(?)", e.user_id)], select: %{}, select_merge: %{^custom_prop => "(none)"}, having: fragment("uniq(?)", e.user_id) > 0 @@ -85,14 +83,16 @@ defmodule Plausible.Stats.Breakdown do end trace(query, property, metrics) - results = breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) - zipped = zip_results(none_result, results, custom_prop, metrics) + results = + breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) + |> Kernel.++(none_result) + |> Enum.sort_by(& &1[sorting_key(metrics)], :desc) - if Enum.find_index(zipped, fn value -> value[custom_prop] == "(none)" end) == limit do - Enum.slice(zipped, 0..(limit - 1)) + if Enum.find_index(results, fn value -> value[custom_prop] == "(none)" end) == limit do + Enum.slice(results, 0..(limit - 1)) else - zipped + results end end @@ -157,18 +157,6 @@ defmodule Plausible.Stats.Breakdown do end defp zip_results(event_result, session_result, property, metrics) do - sort_by = if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) - - property = - if is_binary(property) do - property - |> String.trim_leading("event:") - |> String.trim_leading("visit:") - |> String.trim_leading("props:") - else - property - end - null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{}) prop_values = @@ -179,12 +167,19 @@ defmodule Plausible.Stats.Breakdown do event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{} session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{} - Map.merge(null_row, event_row) + null_row + |> Map.merge(event_row) |> Map.merge(session_row) end) - |> Enum.sort_by(fn row -> row[sort_by] end, :desc) + |> Enum.sort_by(& &1[sorting_key(metrics)], :desc) end + defp include_none_result?({:is, value}), do: value == "(none)" + defp include_none_result?({:is_not, "(none)"}), do: false + defp include_none_result?({:member, values}), do: Enum.member?(values, "(none)") + defp include_none_result?({:not_member, values}), do: !Enum.member?(values, "(none)") + defp include_none_result?(_), do: true + defp breakdown_sessions(_, _, _, [], _), do: [] defp breakdown_sessions(site, query, property, metrics, pagination) do @@ -560,6 +555,10 @@ defmodule Plausible.Stats.Breakdown do ) end + defp sorting_key(metrics) do + if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) + end + defp transform_keys(results, keys_to_replace) do Enum.map(results, fn map -> Enum.map(map, fn {key, val} -> diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex new file mode 100644 index 000000000..ad87471b7 --- /dev/null +++ b/lib/plausible/stats/custom_props.ex @@ -0,0 +1,51 @@ +defmodule Plausible.Stats.CustomProps do + @moduledoc """ + Module for querying user defined 'custom properties'. + """ + + alias Plausible.Stats.Query + use Plausible.ClickhouseRepo + import Plausible.Stats.Base + + @doc """ + Returns a breakdown of event names with all existing custom + properties for each event name. + """ + def props_for_all_event_names(site, query) do + from(e in base_event_query(site, query), + inner_lateral_join: meta in fragment("meta"), + group_by: e.name, + select: {e.name, fragment("groupArray(?)", meta.key)}, + distinct: true + ) + |> ClickhouseRepo.all() + |> Enum.into(%{}) + end + + @doc """ + Expects a single goal filter in the query. Returns a list of custom + props for that goal. Works for both custom event and pageview goals. + """ + def props_for_goal(site, query) do + case query.filters["event:goal"] do + {:is, _} -> fetch_prop_names(site, query) + {:matches, _} -> fetch_prop_names(site, query) + _ -> [] + end + end + + defp fetch_prop_names(site, query) do + case Query.get_filter_by_prefix(query, "event:props:") do + {"event:props:" <> key, _} -> + [key] + + _ -> + from(e in base_event_query(site, query), + inner_lateral_join: meta in fragment("meta"), + select: meta.key, + distinct: true + ) + |> ClickhouseRepo.all() + end + end +end diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 1efe570e6..bc6701c0c 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -144,39 +144,37 @@ defmodule Plausible.Stats.FilterSuggestions do limit: 25 ) - q = - case Query.get_filter_by_prefix(query, "event:props") do - {"event:props:", {:is, val}} -> from([e, m] in q, where: m.value == ^val) - _ -> q - end - ClickhouseRepo.all(q) - |> Enum.filter(fn suggestion -> suggestion != "" end) |> wrap_suggestions() end def filter_suggestions(site, query, "prop_value", filter_search) do filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" - q = + {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props") + + none_q = + from(e in base_event_query(site, Query.remove_event_filters(query, [:props])), + left_lateral_join: meta in "meta", + as: :meta, + select: "(none)", + where: fragment("not has(?, ?)", field(e, :"meta.key"), ^key), + limit: 1 + ) + + search_q = from(e in base_event_query(site, query), inner_lateral_join: meta in "meta", as: :meta, select: meta.value, - where: fragment("? ilike ?", meta.value, ^filter_query), + where: meta.key == ^key and fragment("? ilike ?", meta.value, ^filter_query), group_by: meta.value, order_by: [desc: fragment("count(*)")], limit: 25 ) - q = - case Query.get_filter_by_prefix(query, "event:props") do - {"event:props:" <> key, _filter} -> from([e, m] in q, where: m.key == ^key) - _ -> q - end - - ClickhouseRepo.all(q) - |> Enum.filter(fn suggestion -> suggestion != "" end) + ClickhouseRepo.all(none_q) + |> Kernel.++(ClickhouseRepo.all(search_q)) |> wrap_suggestions() end diff --git a/lib/plausible/stats/mod.ex b/lib/plausible/stats/mod.ex index 97f329a37..704e05cb4 100644 --- a/lib/plausible/stats/mod.ex +++ b/lib/plausible/stats/mod.ex @@ -3,7 +3,6 @@ defmodule Plausible.Stats do defdelegate aggregate(site, query, metrics), to: Plausible.Stats.Aggregate defdelegate timeseries(site, query, metrics), to: Plausible.Stats.Timeseries defdelegate current_visitors(site), to: Plausible.Stats.CurrentVisitors - defdelegate props(site, query), to: Plausible.Stats.Props defdelegate filter_suggestions(site, query, filter_name, filter_search), to: Plausible.Stats.FilterSuggestions diff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.ex index cc8b6856d..dfb06a459 100644 --- a/lib/plausible/stats/props.ex +++ b/lib/plausible/stats/props.ex @@ -1,6 +1,4 @@ defmodule Plausible.Stats.Props do - use Plausible.ClickhouseRepo - import Plausible.Stats.Base @event_props ["event:page", "event:page_match", "event:name", "event:goal"] @session_props [ "visit:source", @@ -28,36 +26,4 @@ defmodule Plausible.Stats.Props do def valid_prop?(prop) when prop in @session_props, do: true def valid_prop?("event:props:" <> prop) when byte_size(prop) > 0, do: true def valid_prop?(_), do: false - - def props(site, query) do - prop_filter = - Enum.find(query.filters, fn {key, _} -> String.starts_with?(key, "event:props:") end) - - case prop_filter do - {"event:props:" <> key, {_, "(none)"}} -> - {_, {_, goal_name}} = query.filters["event:goal"] - %{goal_name => [key]} - - {"event:props:" <> _key, _} -> - ClickhouseRepo.all( - from [e, meta: meta] in base_event_query(site, query), - select: {e.name, meta.key}, - distinct: true - ) - |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc -> - Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end) - end) - - nil -> - ClickhouseRepo.all( - from e in base_event_query(site, query), - inner_lateral_join: meta in fragment("meta"), - select: {e.name, meta.key}, - distinct: true - ) - |> Enum.reduce(%{}, fn {goal_name, meta_key}, acc -> - Map.update(acc, goal_name, [meta_key], fn list -> [meta_key | list] end) - end) - end - end end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 8e01aabc6..58f7226da 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do use PlausibleWeb, :controller use Plausible.Repo use Plug.ErrorHandler - alias Plausible.Stats.{Query, Props} + alias Plausible.Stats.{Query, CustomProps} def realtime_visitors(conn, _params) do site = conn.assigns[:site] @@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do results = if property == "event:goal" do - prop_names = Props.props(site, query) + prop_names = CustomProps.props_for_all_event_names(site, query) Enum.map(results, fn row -> Map.put(row, "props", prop_names[row[:goal]] || []) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 9366da680..92cf9cea5 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController do use Plausible.Repo use Plug.ErrorHandler alias Plausible.Stats - alias Plausible.Stats.{Query, Filters, Comparisons} + alias Plausible.Stats.{Query, Filters, Comparisons, CustomProps} require Logger @@ -1047,13 +1047,6 @@ defmodule PlausibleWeb.Api.StatsController do %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) - prop_names = - if query.filters["event:goal"] do - Stats.props(site, query) - else - %{} - end - conversions = Stats.breakdown(site, query, "event:goal", [:visitors, :events], {100, 1}) |> transform_keys(%{ @@ -1063,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController do }) |> Enum.map(fn goal -> goal - |> Map.put(:prop_names, prop_names[goal[:name]]) + |> Map.put(:prop_names, CustomProps.props_for_goal(site, query)) |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal[:unique_conversions])) end) @@ -1111,15 +1104,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site, params) |> Filters.add_prefix() - prop_names = - if query.filters["event:goal"] do - {_, {_, goal}} = query.filters["event:goal"] - - Stats.props(site, query) - |> Map.get(goal, []) - else - [] - end + prop_names = Plausible.Stats.CustomProps.props_for_goal(site, query) values = prop_names diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index f60d3d762..fdc9047a3 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -1061,44 +1061,27 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do describe "breakdown by event:goal" do test "custom properties from custom events are returned", %{conn: conn, site: site} do - insert(:goal, %{site: site, event_name: "404"}) insert(:goal, %{site: site, event_name: "Purchase"}) insert(:goal, %{site: site, page_path: "/test"}) populate_stats(site, [ - build(:pageview, - timestamp: ~N[2021-01-01 00:00:00], - pathname: "/test" - ), build(:pageview, timestamp: ~N[2021-01-01 00:00:01], pathname: "/test", "meta.key": ["method"], "meta.value": ["HTTP"] ), - build(:event, - name: "404", - timestamp: ~N[2021-01-01 00:00:02], - "meta.key": ["method"], - "meta.value": ["HTTP"] - ), build(:event, name: "Purchase", - timestamp: ~N[2021-01-01 00:00:02], - "meta.key": ["method"], - "meta.value": ["HTTPS"] - ), - build(:event, - name: "404", timestamp: ~N[2021-01-01 00:00:03], "meta.key": ["OS", "method"], "meta.value": ["Linux", "HTTP"] ), build(:event, - name: "404", - timestamp: ~N[2021-01-01 00:00:04], - "meta.key": ["version"], - "meta.value": ["1"] + name: "Purchase", + timestamp: ~N[2021-01-01 00:00:03], + "meta.key": ["OS"], + "meta.value": ["Linux"] ) ]) @@ -1110,28 +1093,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do "property" => "event:goal" }) - res = - Enum.map(json_response(conn, 200)["results"], fn item -> - Map.update(item, "props", [], fn x -> Enum.sort(x) end) - end) - - assert res == [ + assert [ %{ - "goal" => "404", - "props" => ["OS", "method", "version"], - "visitors" => 3 + "goal" => "Purchase", + "props" => props, + "visitors" => 2 }, %{ "goal" => "Visit /test", "props" => [], - "visitors" => 2 - }, - %{ - "goal" => "Purchase", - "props" => ["method"], "visitors" => 1 } - ] + ] = json_response(conn, 200)["results"] + + assert "method" in props + assert "OS" in props end end @@ -1481,6 +1457,94 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do } end + test "IN filter for event:props:*", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + "meta.key": ["browser"], + "meta.value": ["Safari"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "visit:browser", + "filters" => "event:props:browser == Chrome|Safari" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"browser" => "Chrome", "visitors" => 2}, + %{"browser" => "Safari", "visitors" => 1} + ] + } + end + + test "IN filter for event:props:* including (none) value", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Safari", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + browser: "Firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "visit:browser", + "filters" => "event:props:browser == Chrome|(none)" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"browser" => "Chrome", "visitors" => 2}, + %{"browser" => "Safari", "visitors" => 1} + ] + } + end + test "can use a is_not filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, browser: "Chrome"), diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index e6a0626c2..f62fc3487 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -37,14 +37,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Signup", "unique_conversions" => 2, "total_conversions" => 3, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 }, %{ "name" => "Visit /register", "unique_conversions" => 2, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 } ] @@ -86,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Payment", "unique_conversions" => 1, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 } ] @@ -127,7 +127,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Payment", "unique_conversions" => 2, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 66.7 } ] @@ -166,7 +166,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Payment", "unique_conversions" => 2, "total_conversions" => 3, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 66.7 } ] @@ -207,7 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Payment", "unique_conversions" => 2, "total_conversions" => 3, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 66.7 } ] @@ -249,6 +249,76 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end + test "returns only the prop name for the property in filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Payment", + "meta.key": ["logged_in"], + "meta.value": ["true"] + ), + build(:event, + name: "Payment", + "meta.key": ["logged_in"], + "meta.value": ["false"] + ), + build(:event, + name: "Payment", + "meta.key": ["author"], + "meta.value": ["John"] + ) + ]) + + insert(:goal, %{site: site, event_name: "Payment"}) + + filters = Jason.encode!(%{goal: "Payment", props: %{"logged_in" => "true|(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Payment", + "unique_conversions" => 2, + "total_conversions" => 2, + "prop_names" => ["logged_in"], + "conversion_rate" => 66.7 + } + ] + end + + test "returns prop_names=nil when goal :member + property filter are applied at the same time", + %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Payment", + "meta.key": ["logged_in"], + "meta.value": ["true"] + ), + build(:event, + name: "Payment", + "meta.key": ["logged_in"], + "meta.value": ["false"] + ), + build(:event, + name: "Payment", + "meta.key": ["author"], + "meta.value": ["John"] + ) + ]) + + insert(:goal, %{site: site, event_name: "Payment"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + filters = Jason.encode!(%{goal: "Payment|Signup", props: %{"logged_in" => "true|(none)"}}) + conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}") + + assert [%{"prop_names" => []}] = json_response(conn, 200) + end + test "can filter by multiple mixed goals", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/"), @@ -275,14 +345,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Signup", "unique_conversions" => 2, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 }, %{ "name" => "Visit /register", "unique_conversions" => 1, "total_conversions" => 1, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 16.7 } ] @@ -316,14 +386,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "CTA", "unique_conversions" => 1, "total_conversions" => 1, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 16.7 }, %{ "name" => "Visit /register", "unique_conversions" => 1, "total_conversions" => 1, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 16.7 } ] @@ -356,14 +426,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Visit /blog**", "unique_conversions" => 2, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 }, %{ "name" => "Signup", "unique_conversions" => 1, "total_conversions" => 1, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 16.7 } ] @@ -397,14 +467,14 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "name" => "Visit /ano**", "unique_conversions" => 2, "total_conversions" => 2, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 33.3 }, %{ "name" => "CTA", "unique_conversions" => 1, "total_conversions" => 1, - "prop_names" => nil, + "prop_names" => [], "conversion_rate" => 16.7 } ] @@ -519,6 +589,410 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do ] end + test "does not return (none) value in property breakdown with is filter on prop_value", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "0"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "0", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 50.0 + } + ] + end + + test "returns only (none) value in property breakdown with is (none) filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "(none)"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "(none)", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 50.0 + } + ] + end + + test "returns (none) value in property breakdown with is_not filter on prop_value", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "!0"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "20", + "unique_conversions" => 2, + "total_conversions" => 2, + "conversion_rate" => 50.0 + }, + %{ + "name" => "(none)", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 25.0 + } + ] + end + + test "does not return (none) value in property breakdown with is_not (none) filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "!(none)"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "0", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 50.0 + } + ] + end + + test "does not return (none) value in property breakdown with member filter on prop_value", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["1"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["1"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "0|1"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "1", + "unique_conversions" => 2, + "total_conversions" => 2, + "conversion_rate" => 50.0 + }, + %{ + "name" => "0", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 25.0 + } + ] + end + + test "returns (none) value in property breakdown with member filter including a (none) value", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["1"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["1"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "1|(none)"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "1", + "unique_conversions" => 2, + "total_conversions" => 2, + "conversion_rate" => 50.0 + }, + %{ + "name" => "(none)", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 25.0 + } + ] + end + + test "returns (none) value in property breakdown with not_member filter on prop_value", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0.01"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "!0|0.01"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "20", + "unique_conversions" => 2, + "total_conversions" => 2, + "conversion_rate" => 40.0 + }, + %{ + "name" => "(none)", + "unique_conversions" => 1, + "total_conversions" => 1, + "conversion_rate" => 20.0 + } + ] + end + + test "does not return (none) value in property breakdown with not_member filter including a (none) value", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["0"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, + name: "Purchase", + "meta.key": ["cost"], + "meta.value": ["20"] + ), + build(:event, name: "Purchase") + ]) + + insert(:goal, %{site: site, event_name: "Purchase"}) + + filters = + Jason.encode!(%{ + goal: "Purchase", + props: %{cost: "!0|(none)"} + }) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/cost?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "20", + "unique_conversions" => 2, + "total_conversions" => 2, + "conversion_rate" => 50.0 + } + ] + end + + test "returns property breakdown with a pageview goal filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/register"), + build(:pageview, pathname: "/register", "meta.key": ["variant"], "meta.value": ["A"]), + build(:pageview, pathname: "/register", "meta.key": ["variant"], "meta.value": ["A"]) + ]) + + insert(:goal, %{site: site, page_path: "/register"}) + filters = Jason.encode!(%{goal: "Visit /register"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/property/variant?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "unique_conversions" => 2, + "name" => "A", + "total_conversions" => 2, + "conversion_rate" => 50.0 + }, + %{ + "unique_conversions" => 1, + "name" => "(none)", + "total_conversions" => 1, + "conversion_rate" => 25.0 + } + ] + end + test "property breakdown with prop filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, user_id: 1), @@ -700,56 +1174,93 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do "unique_conversions" => 8, "name" => "Visit /**", "total_conversions" => 8, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 37.5, "unique_conversions" => 3, "name" => "Visit /*", "total_conversions" => 3, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 37.5, "unique_conversions" => 3, "name" => "Visit /signup/**", "total_conversions" => 3, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 25.0, "unique_conversions" => 2, "name" => "Visit /billing**/success", "total_conversions" => 2, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 25.0, "unique_conversions" => 2, "name" => "Visit /reg*", "total_conversions" => 2, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 12.5, "unique_conversions" => 1, "name" => "Visit /billing*/success", "total_conversions" => 1, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 12.5, "unique_conversions" => 1, "name" => "Visit /register", "total_conversions" => 1, - "prop_names" => nil + "prop_names" => [] }, %{ "conversion_rate" => 12.5, "unique_conversions" => 1, "name" => "Visit /signup/*", "total_conversions" => 1, - "prop_names" => nil + "prop_names" => [] + } + ] + end + + test "returns prop names when filtered by glob goal", %{conn: conn, site: site} do + insert(:goal, %{site: site, page_path: "/register**"}) + + populate_stats(site, [ + build(:pageview, + pathname: "/register", + "meta.key": ["logged_in"], + "meta.value": ["false"], + timestamp: ~N[2019-07-01 23:00:00] + ), + build(:pageview, + pathname: "/register-success", + "meta.key": ["logged_in", "author"], + "meta.value": ["true", "John"], + timestamp: ~N[2019-07-01 23:00:00] + ) + ]) + + filters = Jason.encode!(%{goal: "Visit /register**"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/conversions?period=day&date=2019-07-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "conversion_rate" => 100.0, + "unique_conversions" => 2, + "name" => "Visit /register**", + "total_conversions" => 2, + "prop_names" => ["logged_in", "author"] } ] end diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index c549bb516..720372edc 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -307,6 +307,94 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top pages with :not_member filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/safari", + "meta.key": ["browser"], + "meta.value": ["Safari"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/firefox", + "meta.key": ["browser"], + "meta.value": ["Firefox"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/firefox", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"browser" => "!Chrome|Safari"}}) + + conn = + get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "/firefox", + "visitors" => 2 + } + ] + end + + test "returns top pages with :not_member filter on custom pageview props including (none) value", + %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/chrome", + "meta.key": ["browser"], + "meta.value": ["Chrome"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/safari", + "meta.key": ["browser"], + "meta.value": ["Safari"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/no-browser-prop", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"browser" => "!Chrome|(none)"}}) + + conn = + get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "/safari", + "visitors" => 1 + } + ] + end + test "calculates bounce_rate and time_on_page for pages filtered by page path", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs index ffa727973..6056dd6d8 100644 --- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs @@ -234,7 +234,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do describe "suggestions for props" do setup [:create_user, :log_in, :create_new_site] - test "returns suggestions for prop key with no filter", %{conn: conn, site: site} do + test "returns suggestions for prop key ordered by count", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, "meta.key": ["author"], @@ -242,29 +242,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, - "meta.key": ["logged_in"], - "meta.value": ["false"], + "meta.key": ["author"], + "meta.value": ["Uku Taht"], timestamp: ~N[2022-01-01 00:00:00] ), - build(:pageview, - "meta.key": ["dark_mode"], - "meta.value": ["true"], - timestamp: ~N[2022-01-01 00:00:00] - ) - ]) - - conn = - get(conn, "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01") - - assert json_response(conn, 200) |> Enum.map(& &1["value"]) |> Enum.sort() == [ - "author", - "dark_mode", - "logged_in" - ] - end - - test "returns suggestions for prop key with value filter", %{conn: conn, site: site} do - populate_stats(site, [ build(:pageview, "meta.key": ["author"], "meta.value": ["Uku Taht"], @@ -272,7 +253,12 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do ), build(:pageview, "meta.key": ["logged_in"], - "meta.value": ["true"], + "meta.value": ["false"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["logged_in"], + "meta.value": ["false"], timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, @@ -282,21 +268,18 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do ) ]) - filters = Jason.encode!(%{props: %{"": "true"}}) - conn = - get( - conn, - "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01&filters=#{filters}" - ) + get(conn, "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01") - assert json_response(conn, 200) |> Enum.map(& &1["value"]) |> Enum.sort() == [ - "dark_mode", - "logged_in" + assert json_response(conn, 200) == [ + %{"label" => "author", "value" => "author"}, + %{"label" => "logged_in", "value" => "logged_in"}, + %{"label" => "dark_mode", "value" => "dark_mode"} ] end - test "returns suggestions for prop value with no filter", %{conn: conn, site: site} do + test "returns suggestions for prop value ordered by count, but (none) value is always first", + %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, "meta.key": ["author"], @@ -304,28 +287,39 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, - "meta.key": ["logged_in"], - "meta.value": ["false"], + "meta.key": ["author"], + "meta.value": ["Uku Taht"], + timestamp: ~N[2022-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["Marko Saric"], timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, - "meta.key": ["dark_mode"], - "meta.value": ["true"], timestamp: ~N[2022-01-01 00:00:00] ) ]) + filters = Jason.encode!(%{props: %{author: "!(none)"}}) + conn = - get(conn, "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01") + get( + conn, + "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01&filters=#{filters}" + ) - assert json_response(conn, 200) |> Enum.map(& &1["value"]) |> Enum.sort() == [ - "Uku Taht", - "false", - "true" + assert json_response(conn, 200) == [ + %{"label" => "(none)", "value" => "(none)"}, + %{"label" => "Uku Taht", "value" => "Uku Taht"}, + %{"label" => "Marko Saric", "value" => "Marko Saric"} ] end - test "returns suggestions for prop value with filter on prop key", %{conn: conn, site: site} do + test "does not show (none) value suggestion when all events have that prop_key", %{ + conn: conn, + site: site + } do populate_stats(site, [ build(:pageview, "meta.key": ["author"], @@ -333,13 +327,13 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, - "meta.key": ["logged_in"], - "meta.value": ["false"], + "meta.key": ["author"], + "meta.value": ["Uku Taht"], timestamp: ~N[2022-01-01 00:00:00] ), build(:pageview, - "meta.key": ["dark_mode"], - "meta.value": ["true"], + "meta.key": ["author"], + "meta.value": ["Marko Saric"], timestamp: ~N[2022-01-01 00:00:00] ) ]) @@ -352,7 +346,10 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do "/api/stats/#{site.domain}/suggestions/prop_value?period=day&date=2022-01-01&filters=#{filters}" ) - assert json_response(conn, 200) |> Enum.map(& &1["value"]) |> Enum.sort() == ["Uku Taht"] + assert json_response(conn, 200) == [ + %{"label" => "Uku Taht", "value" => "Uku Taht"}, + %{"label" => "Marko Saric", "value" => "Marko Saric"} + ] end test "when date is borked, bad request is returned", %{