From 1b95433d6ed1eb3b1c0e0a74cc847bcfcf379435 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Mon, 22 Jan 2024 12:23:59 +0200 Subject: [PATCH] Support `contains`/`matches` for custom properties (#3687) * Allow `matches` operator to work in BE for custom props Note: No FE support yet, needs further testing * feat: allow choosing `contains` for property filters in the UI * no autocomplete on prop values if `contains` for consistency * CHANGELOG.md * Fix: Handle (none) property in property breakdowns when using matching When matching we should always exclude (none) --- CHANGELOG.md | 1 + .../stats/modals/prop-filter-modal.js | 17 +++++++-- assets/js/dashboard/util/filters.js | 6 ++- lib/plausible/stats/base.ex | 10 +++++ lib/plausible/stats/breakdown.ex | 1 + .../custom_prop_breakdown_test.exs | 36 ++++++++++++++++++ .../api/stats_controller/pages_test.exs | 37 +++++++++++++++++++ 7 files changed, 104 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ab4b663..5b7ea5645 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to this project will be documented in this file. ### Added +- Allow filtering with `contains`/`matches` operator for custom properties - Add `referrers.csv` to CSV export - Add a new Properties section in the dashboard to break down by custom properties - Add `custom_props.csv` to CSV export (almost the same as the old `prop_breakdown.csv`, but has different column headers, and includes props for pageviews too, not only custom events) diff --git a/assets/js/dashboard/stats/modals/prop-filter-modal.js b/assets/js/dashboard/stats/modals/prop-filter-modal.js index f0d28f2eb..80faab37f 100644 --- a/assets/js/dashboard/stats/modals/prop-filter-modal.js +++ b/assets/js/dashboard/stats/modals/prop-filter-modal.js @@ -40,11 +40,15 @@ function PropFilterModal(props) { const fetchPropValueOptions = useCallback(() => { return (input) => { + if (formState.prop_value?.type === FILTER_TYPES.contains) { + return Promise.resolve([]) + } + const propKey = formState.prop_key?.value const updatedQuery = { ...query, filters: { ...query.filters, props: {[propKey]: '!(none)'} } } return api.get(apiPath(props.site, "/suggestions/prop_value"), updatedQuery, { q: input.trim() }) } - }, [formState.prop_key]) + }, [formState.prop_key, formState.prop_value]) function onPropKeySelect() { return (selectedOptions) => { @@ -86,7 +90,14 @@ function PropFilterModal(props) {
- +
) @@ -167,4 +178,4 @@ function PropFilterModal(props) { ) } -export default withRouter(PropFilterModal) \ No newline at end of file +export default withRouter(PropFilterModal) diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.js index 7aaa30476..61ad7dae0 100644 --- a/assets/js/dashboard/util/filters.js +++ b/assets/js/dashboard/util/filters.js @@ -10,6 +10,10 @@ export const FILTER_GROUPS = { 'props': ['prop_key', 'prop_value'] } +export const ALLOW_FREE_CHOICE = new Set( + FILTER_GROUPS['page'].concat(FILTER_GROUPS['utm']).concat(['prop_value']) +) + export const FILTER_TYPES = { isNot: 'is not', contains: 'contains', @@ -27,7 +31,7 @@ export function supportsIsNot(filterName) { } export function isFreeChoiceFilter(filterName) { - return FILTER_GROUPS['page'].concat(FILTER_GROUPS['utm']).includes(filterName) + return ALLOW_FREE_CHOICE.has(filterName) } // As of March 2023, Safari does not support negative lookbehind regexes. In case it throws an error, falls back to plain | matching. This means diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index e5cfa3ba9..b5e1cae99 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -142,6 +142,16 @@ defmodule Plausible.Stats.Base do ) end + {"event:props:" <> prop_name, {:matches, value}} -> + regex = page_regex(value) + + from( + e in q, + left_array_join: meta in "meta", + as: :meta, + where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex) + ) + {"event:props:" <> prop_name, {:member, values}} -> none_value_included = Enum.member?(values, "(none)") diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index c3716041b..2cecadfb0 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -189,6 +189,7 @@ defmodule Plausible.Stats.Breakdown do defp include_none_result?({:is_not, "(none)"}), do: false defp include_none_result?({:member, values}), do: Enum.member?(values, "(none)") defp include_none_result?({:not_member, values}), do: !Enum.member?(values, "(none)") + defp include_none_result?({:matches, _}), do: false defp include_none_result?(_), do: true defp breakdown_sessions(_, _, _, [], _), do: [] diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs index 46c6b1c61..08f62701e 100644 --- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs @@ -1046,6 +1046,42 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do } ] end + + test "returns prop-breakdown with a prop_value matching filter", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, "meta.key": ["key"], "meta.value": ["foo"]), + build(:pageview, "meta.key": ["key"], "meta.value": ["bar"]), + build(:pageview, "meta.key": ["key"], "meta.value": ["bar"]), + build(:pageview, "meta.key": ["key"], "meta.value": ["foobar"]), + build(:pageview) + ]) + + filters = Jason.encode!(%{props: %{key: "~bar"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{ + "visitors" => 2, + "name" => "bar", + "events" => 2, + "percentage" => 66.7 + }, + %{ + "visitors" => 1, + "name" => "foobar", + "events" => 1, + "percentage" => 33.3 + } + ] + end end describe "GET /api/stats/:domain/custom-prop-values/:prop_key - for a Growth subscription" do diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 62bb5c849..339a4bab3 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -75,6 +75,43 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top pages with :matches filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/1", + "meta.key": ["prop"], + "meta.value": ["bar"] + ), + build(:pageview, + pathname: "/2", + "meta.key": ["prop"], + "meta.value": ["foobar"] + ), + build(:pageview, + pathname: "/3", + "meta.key": ["prop"], + "meta.value": ["baar"] + ), + build(:pageview, + pathname: "/4", + "meta.key": ["another"], + "meta.value": ["bar"] + ), + build(:pageview, pathname: "/5") + ]) + + filters = Jason.encode!(%{props: %{"prop" => "~bar"}}) + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{"visitors" => 1, "name" => "/1"}, + %{"visitors" => 1, "name" => "/2"} + ] + end + test "calculates bounce_rate and time_on_page with :is filter on custom pageview props", %{conn: conn, site: site} do populate_stats(site, [