From 11c5d3b251a02c367828d3a83f3b4ec7707e2f91 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 23 Jan 2024 09:32:49 +0200 Subject: [PATCH] Remove ARRAY JOINs (#3708) * Use array indexing instead of left joins This has multiple benefits: - Allows us to more easily add support for multiple custom property filters - Performance: Increased performance of some queries by ~20% Work left to do: - Check tests coverage - Create macros for repeated fragments - Quote and refer to right tables in fragments/macros * Use macros to clean up code * Use existing atoms to avoid memory leaks --- lib/plausible/stats/base.ex | 56 ++++++++++------------- lib/plausible/stats/breakdown.ex | 33 +++---------- lib/plausible/stats/clickhouse.ex | 5 +- lib/plausible/stats/filter_suggestions.ex | 19 ++++---- lib/plausible/stats/fragments.ex | 46 +++++++++++++++++++ 5 files changed, 89 insertions(+), 70 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index b5e1cae99..e9208088e 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -1,6 +1,8 @@ defmodule Plausible.Stats.Base do use Plausible.ClickhouseRepo use Plausible + use Plausible.Stats.Fragments + alias Plausible.Stats.{Query, Filters} alias Plausible.Timezones import Ecto.Query @@ -114,14 +116,12 @@ defmodule Plausible.Stats.Base do if value == "(none)" do from( e in q, - where: fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name) + where: not has_key(e, :meta, ^prop_name) ) else from( e in q, - array_join: meta in "meta", - as: :meta, - where: meta.key == ^prop_name and meta.value == ^value + where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value ) end @@ -129,16 +129,13 @@ defmodule Plausible.Stats.Base do if value == "(none)" do from( e in q, - where: fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) + where: has_key(e, :meta, ^prop_name) ) else from( e in q, - left_array_join: meta in "meta", - as: :meta, where: - (meta.key == ^prop_name and meta.value != ^value) or - fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name) + not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value ) end @@ -147,9 +144,9 @@ defmodule Plausible.Stats.Base do from( e in q, - left_array_join: meta in "meta", - as: :meta, - where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex) + where: + has_key(e, :meta, ^prop_name) and + fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex) ) {"event:props:" <> prop_name, {:member, values}} -> @@ -157,12 +154,9 @@ defmodule Plausible.Stats.Base do from( e in q, - left_array_join: meta in "meta", - as: :meta, where: - (meta.key == ^prop_name and meta.value in ^values) or - (^none_value_included and - fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)) + (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or + (^none_value_included and not has_key(e, :meta, ^prop_name)) ) {"event:props:" <> prop_name, {:not_member, values}} -> @@ -170,14 +164,13 @@ defmodule Plausible.Stats.Base do from( e in q, - left_array_join: meta in "meta", - as: :meta, where: - (meta.key == ^prop_name and meta.value not in ^values) or - (^none_value_included and fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) and - meta.value not in ^values) or - (not (^none_value_included) and - fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)) + (has_key(e, :meta, ^prop_name) and + get_by_key(e, :meta, ^prop_name) not in ^values) or + (^none_value_included and + has_key(e, :meta, ^prop_name) and + get_by_key(e, :meta, ^prop_name) not in ^values) or + (not (^none_value_included) and not has_key(e, :meta, ^prop_name)) ) _ -> @@ -240,34 +233,31 @@ defmodule Plausible.Stats.Base do def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do from( s in sessions_q, - where: fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) + where: not has_key(s, :entry_meta, ^prop_name) ) end def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do from( s in sessions_q, - array_join: meta in "entry_meta", - as: :meta, - where: meta.key == ^prop_name and meta.value == ^value + where: + has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value ) end def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do from( s in sessions_q, - where: fragment("has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) + where: has_key(s, :entry_meta, ^prop_name) ) end def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do from( s in sessions_q, - left_array_join: meta in "entry_meta", - as: :meta, where: - (meta.key == ^prop_name and meta.value != ^value) or - fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) + not has_key(s, :entry_meta, ^prop_name) or + get_by_key(s, :entry_meta, ^prop_name) != ^value ) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 0d5377485..cf3e93d3c 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -1,6 +1,8 @@ defmodule Plausible.Stats.Breakdown do use Plausible.ClickhouseRepo use Plausible + use Plausible.Stats.Fragments + import Plausible.Stats.{Base, Imported, Util} require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.Query @@ -397,37 +399,16 @@ defmodule Plausible.Stats.Breakdown do |> Map.new() end - defp joins_table?(ecto_q, table) do - Enum.any?( - ecto_q.joins, - fn - %Ecto.Query.JoinExpr{source: {^table, _}} -> true - _ -> false - end - ) - end - defp do_group_by( %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q, "event:props:" <> prop ) do - q = - if joins_table?(q, "meta") do - q - else - from( - e in q, - array_join: meta in fragment("meta"), - as: :meta - ) - end - from( - [e, meta: meta] in q, - where: meta.key == ^prop, - group_by: meta.value, - select_merge: %{^prop => meta.value}, - order_by: {:asc, meta.value} + e in q, + where: has_key(e, :meta, ^prop), + select_merge: %{^prop => get_by_key(e, :meta, ^prop)}, + group_by: get_by_key(e, :meta, ^prop), + order_by: {:asc, get_by_key(e, :meta, ^prop)} ) end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index a3e2f9a8f..ce116f491 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -538,13 +538,12 @@ defmodule Plausible.Stats.Clickhouse do if val == "(none)" do from( e in q, - where: fragment("not has(meta.key, ?)", ^key) + where: not has_key(e, :meta, ^key) ) else from( e in q, - array_join: meta in fragment("meta"), - where: meta.key == ^key and meta.value == ^val + where: has_key(e, :meta, ^key) and get_by_key(e, :meta, ^key) == ^val ) end else diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 0e5d97986..ae71f2de1 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -1,6 +1,7 @@ defmodule Plausible.Stats.FilterSuggestions do use Plausible.Repo use Plausible.ClickhouseRepo + use Plausible.Stats.Fragments import Plausible.Stats.Base alias Plausible.Stats.Query @@ -154,20 +155,22 @@ defmodule Plausible.Stats.FilterSuggestions do none_q = from(e in base_event_query(site, Query.remove_event_filters(query, [:props])), - left_array_join: meta in "meta", - as: :meta, select: "(none)", - where: fragment("not has(?, ?)", field(e, :"meta.key"), ^key), + where: not has_key(e, :meta, ^key), limit: 1 ) search_q = from(e in base_event_query(site, query), - array_join: meta in "meta", - as: :meta, - select: meta.value, - where: meta.key == ^key and fragment("? ilike ?", meta.value, ^filter_query), - group_by: meta.value, + select: get_by_key(e, :meta, ^key), + where: + has_key(e, :meta, ^key) and + fragment( + "? ilike ?", + get_by_key(e, :meta, ^key), + ^filter_query + ), + group_by: get_by_key(e, :meta, ^key), order_by: [desc: fragment("count(*)")], limit: 25 ) diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex index a4caea33b..ae5ef4e44 100644 --- a/lib/plausible/stats/fragments.ex +++ b/lib/plausible/stats/fragments.ex @@ -97,6 +97,52 @@ defmodule Plausible.Stats.Fragments do end end + @doc """ + Returns whether a key (usually property) exists under `meta.key` array or similar. + + This macro is used for operating on custom properties. + + ## Examples + + `has_key(e, :meta, "some_property_name")` expands to SQL `has(meta.key, "some_property_name")` + """ + defmacro has_key(table, meta_column, key) do + quote do + fragment( + "has(?, ?)", + field(unquote(table), unquote(meta_key_column(meta_column))), + unquote(key) + ) + end + end + + @doc """ + Returns value of a key (usually property) under `meta.value` array or similar. + + This macro is used for operating on custom properties. + Callsites should also check whether key exists first in SQL via `has_key` macro. + + ## Examples + + `get_by_key(e, :meta, "some_property_name")` expands to SQL `meta.value[indexOf(meta.key, "some_property")]` + """ + defmacro get_by_key(table, meta_column, key) do + quote do + fragment( + "?[indexOf(?, ?)]", + field(unquote(table), unquote(meta_value_column(meta_column))), + field(unquote(table), unquote(meta_key_column(meta_column))), + unquote(key) + ) + end + end + + defp meta_key_column(:meta), do: :"meta.key" + defp meta_key_column(:entry_meta), do: :"entry_meta.key" + + defp meta_value_column(:meta), do: :"meta.value" + defp meta_value_column(:entry_meta), do: :"entry_meta.value" + defmacro __using__(_) do quote do import Plausible.Stats.Fragments