From 7f86f6345b21240d307a6c7ab6dfbefa50debaf5 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Wed, 14 Feb 2024 11:26:53 +0100 Subject: [PATCH] WIP: Goal exclusion --- lib/plausible/stats/base.ex | 56 ++++++++ .../stats/filters/stats_api_filter_parser.ex | 7 +- .../stats/dashboard_filter_parser_test.exs | 11 ++ test/plausible/stats/filters_test.exs | 8 ++ .../api/stats_controller/conversions_test.exs | 123 ++++++++++++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 589f28aa8..9bf57658c 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -7,6 +7,8 @@ defmodule Plausible.Stats.Base do alias Plausible.Timezones import Ecto.Query + require Logger + @no_ref "Direct / None" @not_set "(not set)" @@ -65,6 +67,9 @@ defmodule Plausible.Stats.Base do {:is, {:page, path}} -> from(e in q, where: e.pathname == ^path and e.name == "pageview") + {:is_not, {:page, path}} -> + from(e in q, where: e.pathname != ^path) + {:matches, {:page, expr}} -> regex = page_regex(expr) @@ -72,9 +77,19 @@ defmodule Plausible.Stats.Base do where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview" ) + {:does_not_match, {:page, expr}} -> + regex = page_regex(expr) + + from(e in q, + where: fragment("not(match(?, ?))", e.pathname, ^regex) + ) + {:is, {:event, event}} -> from(e in q, where: e.name == ^event) + {:is_not, {:event, event}} -> + from(e in q, where: e.name != ^event) + {:member, clauses} -> {events, pages} = split_goals(clauses) @@ -82,6 +97,13 @@ defmodule Plausible.Stats.Base do where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events ) + {:not_member, clauses} -> + {events, pages} = split_goals(clauses) + + from(e in q, + where: e.pathname not in ^pages and e.name not in ^events + ) + {:matches_member, clauses} -> {events, pages} = split_goals(clauses, &page_regex/1) @@ -106,8 +128,42 @@ defmodule Plausible.Stats.Base do from(e in q, where: ^where_clause) + {:not_matches_member, clauses} -> + {events, pages} = split_goals(clauses, &page_regex/1) + + event_clause = + if Enum.any?(events) do + dynamic( + [x], + fragment("not(multiMatchAny(?, ?))", x.name, ^events) + ) + else + dynamic([x], false) + end + + page_clause = + if Enum.any?(pages) do + dynamic( + [x], + fragment("not(multiMatchAny(?, ?))", x.pathname, ^pages) + ) + else + dynamic([x], false) + end + + where_clause = dynamic([], ^event_clause and ^page_clause) + + from(e in q, where: ^where_clause) + nil -> q + + clause -> + Logger.error( + "Unsupported goal filter clause: #{inspect(clause)}. Falling back to unfiltered query." + ) + + q end q = diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex index b946e5950..3383d9f07 100644 --- a/lib/plausible/stats/filters/stats_api_filter_parser.ex +++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex @@ -18,7 +18,8 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do defp parse_single_filter(str) do case to_kv(str) do ["event:goal" = key, raw_value] -> - {key, parse_goal_filter(raw_value)} + is_negated? = String.contains?(str, "!=") + {key, parse_goal_filter(raw_value, is_negated?)} [key, raw_value] -> is_negated? = String.contains?(str, "!=") @@ -59,7 +60,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do |> Enum.map(&String.trim/1) end - defp parse_goal_filter(value) do + defp parse_goal_filter(value, is_negated?) do is_list? = list_expression?(value) is_wildcard? = wildcard_expression?(value) @@ -72,6 +73,8 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do |> wrap_goal_value() cond do + is_negated? && is_list? -> {:not_member, value} + is_negated? -> {:is_not, value} is_list? && is_wildcard? -> {:matches_member, value} is_list? -> {:member, value} is_wildcard? -> {:matches, value} diff --git a/test/plausible/stats/dashboard_filter_parser_test.exs b/test/plausible/stats/dashboard_filter_parser_test.exs index ad5a5d657..46f2d485a 100644 --- a/test/plausible/stats/dashboard_filter_parser_test.exs +++ b/test/plausible/stats/dashboard_filter_parser_test.exs @@ -11,9 +11,15 @@ defmodule Plausible.Stats.DashboardFilterParserTest do %{"page" => "/"} |> assert_parsed(%{"event:page" => {:is, "/"}}) + %{"page" => "!/"} + |> assert_parsed(%{"event:page" => {:is_not, "/"}}) + %{"goal" => "Signup"} |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}}) + %{"goal" => "!Signup"} + |> assert_parsed(%{"event:goal" => {:is_not, {:event, "Signup"}}}) + %{"goal" => "Visit /blog"} |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}}) @@ -109,6 +115,11 @@ defmodule Plausible.Stats.DashboardFilterParserTest do %{"goal" => "Visit /thank-you|Signup"} |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}}) + + %{"goal" => "!Visit /thank-you|Signup"} + |> assert_parsed(%{ + "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]} + }) end end diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exs index 4f87bd995..bc7f13669 100644 --- a/test/plausible/stats/filters_test.exs +++ b/test/plausible/stats/filters_test.exs @@ -40,6 +40,14 @@ defmodule Plausible.Stats.FiltersTest do |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}}) end + test "negative goals" do + "event:goal!=Signup" + |> assert_parsed(%{"event:goal" => {:is_not, {:event, "Signup"}}}) + + "event:goal!=Signup|Visit /blog" + |> assert_parsed(%{"event:goal" => {:not_member, [event: "Signup", page: "/blog"]}}) + end + test "pageview goal" do "event:goal==Visit /blog" |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}}) diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index a454f59f6..4f5e70b1d 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -472,6 +472,129 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert get_with_filter.(%{goal: "Visit+/register|Visit+/regi**"}) == expected end + test "goal filter negations", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/register"), + build(:pageview, pathname: "/logout"), + build(:event, pathname: "/register", name: "Signup") + ]) + + insert(:goal, %{site: site, page_path: "/register"}) + insert(:goal, %{site: site, page_path: "/logout"}) + insert(:goal, %{site: site, page_path: "/regi*"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + get_with_filter = fn filters -> + path = "/api/stats/#{site.domain}/conversions" + query = "?period=day&filters=#{Jason.encode!(filters)}" + + get(conn, path <> query) + |> json_response(200) + end + + # {:is_not, {:page, path}} filter type + assert get_with_filter.(%{goal: "!Visit /logout"}) == [ + %{"conversion_rate" => 25.0, "events" => 1, "name" => "Signup", "visitors" => 1}, + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /register", + "visitors" => 1 + }, + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /regi*", + "visitors" => 1 + } + ] + + # {:is_not, {:event, event}} filter type + assert get_with_filter.(%{goal: "!Signup"}) == [ + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /register", + "visitors" => 1 + }, + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /logout", + "visitors" => 1 + }, + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /regi*", + "visitors" => 1 + } + ] + + # {:does_not_match, {:page, expr}} filter type + assert get_with_filter.(%{goal: "!Visit /regi*"}) == [ + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /logout", + "visitors" => 1 + } + ] + + # {:not_matches_member, clauses} filter type + assert get_with_filter.(%{goal: "!Visit /logout|Signup"}) == + [ + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /register", + "visitors" => 1 + }, + %{ + "conversion_rate" => 25.0, + "events" => 1, + "name" => "Visit /regi*", + "visitors" => 1 + } + ] + end + + test "goal exclusion with single custom event on two different paths", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:event, pathname: "/register", name: "Signup"), + build(:event, pathname: "/promo/123/signup", name: "Signup") + ]) + + insert(:goal, %{site: site, page_path: "/regi*"}) + insert(:goal, %{site: site, page_path: "/arbitrary"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + get_with_filter = fn filters -> + path = "/api/stats/#{site.domain}/conversions" + query = "?period=day&filters=#{Jason.encode!(filters)}" + + get(conn, path <> query) + |> json_response(200) + end + + assert get_with_filter.(%{goal: "!Visit /arbitrary"}) == [ + %{"conversion_rate" => 100.0, "events" => 2, "name" => "Signup", "visitors" => 2} + ] + + assert get_with_filter.(%{goal: "!Visit /regi*"}) == [ + %{"conversion_rate" => 50.0, "events" => 1, "name" => "Signup", "visitors" => 1} + ] + + assert get_with_filter.(%{goal: "!Signup"}) == [] + end + test "can filter by multiple mixed goals", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/"),