Remove ARRAY JOINs (#3708)

* Use array indexing instead of left joins

This has multiple benefits:
- Allows us to more easily add support for multiple custom property
  filters
- Performance: Increased performance of some queries by ~20%

Work left to do:
- Check tests coverage
- Create macros for repeated fragments
- Quote and refer to right tables in fragments/macros

* Use macros to clean up code

* Use existing atoms to avoid memory leaks
pull/3720/head r2024.01.23-326
Karl-Aksel Puulmann 3 years ago committed by GitHub
parent e2eb861799
commit 11c5d3b251
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -1,6 +1,8 @@
defmodule Plausible.Stats.Base do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.Fragments
alias Plausible.Stats.{Query, Filters}
alias Plausible.Timezones
import Ecto.Query
@ -114,14 +116,12 @@ defmodule Plausible.Stats.Base do
if value == "(none)" do
from(
e in q,
where: fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)
where: not has_key(e, :meta, ^prop_name)
)
else
from(
e in q,
array_join: meta in "meta",
as: :meta,
where: meta.key == ^prop_name and meta.value == ^value
where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value
)
end
@ -129,16 +129,13 @@ defmodule Plausible.Stats.Base do
if value == "(none)" do
from(
e in q,
where: fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name)
where: has_key(e, :meta, ^prop_name)
)
else
from(
e in q,
left_array_join: meta in "meta",
as: :meta,
where:
(meta.key == ^prop_name and meta.value != ^value) or
fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)
not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value
)
end
@ -147,9 +144,9 @@ defmodule Plausible.Stats.Base do
from(
e in q,
left_array_join: meta in "meta",
as: :meta,
where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex)
where:
has_key(e, :meta, ^prop_name) and
fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)
)
{"event:props:" <> prop_name, {:member, values}} ->
@ -157,12 +154,9 @@ defmodule Plausible.Stats.Base do
from(
e in q,
left_array_join: meta in "meta",
as: :meta,
where:
(meta.key == ^prop_name and meta.value in ^values) or
(^none_value_included and
fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))
(has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or
(^none_value_included and not has_key(e, :meta, ^prop_name))
)
{"event:props:" <> prop_name, {:not_member, values}} ->
@ -170,14 +164,13 @@ defmodule Plausible.Stats.Base do
from(
e in q,
left_array_join: meta in "meta",
as: :meta,
where:
(meta.key == ^prop_name and meta.value not in ^values) or
(^none_value_included and fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) and
meta.value not in ^values) or
(not (^none_value_included) and
fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))
(has_key(e, :meta, ^prop_name) and
get_by_key(e, :meta, ^prop_name) not in ^values) or
(^none_value_included and
has_key(e, :meta, ^prop_name) and
get_by_key(e, :meta, ^prop_name) not in ^values) or
(not (^none_value_included) and not has_key(e, :meta, ^prop_name))
)
_ ->
@ -240,34 +233,31 @@ defmodule Plausible.Stats.Base do
def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do
from(
s in sessions_q,
where: fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
where: not has_key(s, :entry_meta, ^prop_name)
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do
from(
s in sessions_q,
array_join: meta in "entry_meta",
as: :meta,
where: meta.key == ^prop_name and meta.value == ^value
where:
has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do
from(
s in sessions_q,
where: fragment("has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
where: has_key(s, :entry_meta, ^prop_name)
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do
from(
s in sessions_q,
left_array_join: meta in "entry_meta",
as: :meta,
where:
(meta.key == ^prop_name and meta.value != ^value) or
fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
not has_key(s, :entry_meta, ^prop_name) or
get_by_key(s, :entry_meta, ^prop_name) != ^value
)
end

@ -1,6 +1,8 @@
defmodule Plausible.Stats.Breakdown do
use Plausible.ClickhouseRepo
use Plausible
use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported, Util}
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.Query
@ -397,37 +399,16 @@ defmodule Plausible.Stats.Breakdown do
|> Map.new()
end
defp joins_table?(ecto_q, table) do
Enum.any?(
ecto_q.joins,
fn
%Ecto.Query.JoinExpr{source: {^table, _}} -> true
_ -> false
end
)
end
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:props:" <> prop
) do
q =
if joins_table?(q, "meta") do
q
else
from(
e in q,
array_join: meta in fragment("meta"),
as: :meta
)
end
from(
[e, meta: meta] in q,
where: meta.key == ^prop,
group_by: meta.value,
select_merge: %{^prop => meta.value},
order_by: {:asc, meta.value}
e in q,
where: has_key(e, :meta, ^prop),
select_merge: %{^prop => get_by_key(e, :meta, ^prop)},
group_by: get_by_key(e, :meta, ^prop),
order_by: {:asc, get_by_key(e, :meta, ^prop)}
)
end

@ -538,13 +538,12 @@ defmodule Plausible.Stats.Clickhouse do
if val == "(none)" do
from(
e in q,
where: fragment("not has(meta.key, ?)", ^key)
where: not has_key(e, :meta, ^key)
)
else
from(
e in q,
array_join: meta in fragment("meta"),
where: meta.key == ^key and meta.value == ^val
where: has_key(e, :meta, ^key) and get_by_key(e, :meta, ^key) == ^val
)
end
else

@ -1,6 +1,7 @@
defmodule Plausible.Stats.FilterSuggestions do
use Plausible.Repo
use Plausible.ClickhouseRepo
use Plausible.Stats.Fragments
import Plausible.Stats.Base
alias Plausible.Stats.Query
@ -154,20 +155,22 @@ defmodule Plausible.Stats.FilterSuggestions do
none_q =
from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),
left_array_join: meta in "meta",
as: :meta,
select: "(none)",
where: fragment("not has(?, ?)", field(e, :"meta.key"), ^key),
where: not has_key(e, :meta, ^key),
limit: 1
)
search_q =
from(e in base_event_query(site, query),
array_join: meta in "meta",
as: :meta,
select: meta.value,
where: meta.key == ^key and fragment("? ilike ?", meta.value, ^filter_query),
group_by: meta.value,
select: get_by_key(e, :meta, ^key),
where:
has_key(e, :meta, ^key) and
fragment(
"? ilike ?",
get_by_key(e, :meta, ^key),
^filter_query
),
group_by: get_by_key(e, :meta, ^key),
order_by: [desc: fragment("count(*)")],
limit: 25
)

@ -97,6 +97,52 @@ defmodule Plausible.Stats.Fragments do
end
end
@doc """
Returns whether a key (usually property) exists under `meta.key` array or similar.
This macro is used for operating on custom properties.
## Examples
`has_key(e, :meta, "some_property_name")` expands to SQL `has(meta.key, "some_property_name")`
"""
defmacro has_key(table, meta_column, key) do
quote do
fragment(
"has(?, ?)",
field(unquote(table), unquote(meta_key_column(meta_column))),
unquote(key)
)
end
end
@doc """
Returns value of a key (usually property) under `meta.value` array or similar.
This macro is used for operating on custom properties.
Callsites should also check whether key exists first in SQL via `has_key` macro.
## Examples
`get_by_key(e, :meta, "some_property_name")` expands to SQL `meta.value[indexOf(meta.key, "some_property")]`
"""
defmacro get_by_key(table, meta_column, key) do
quote do
fragment(
"?[indexOf(?, ?)]",
field(unquote(table), unquote(meta_value_column(meta_column))),
field(unquote(table), unquote(meta_key_column(meta_column))),
unquote(key)
)
end
end
defp meta_key_column(:meta), do: :"meta.key"
defp meta_key_column(:entry_meta), do: :"entry_meta.key"
defp meta_value_column(:meta), do: :"meta.value"
defp meta_value_column(:entry_meta), do: :"entry_meta.value"
defmacro __using__(_) do
quote do
import Plausible.Stats.Fragments

Loading…
Cancel
Save