Reducing JOINs in queries (#3966)
* Move experimental_session_count? logic to within query object * WIP new querying system for deciding what tables to query * both -> either * Include sample_percent in both tables * Remove a hanging TODO * Allow filtering by visit props on event queries if flag is on * Make default sessions join more conditional * Simplify events_join_sessions? * Add some TODOs * Fix assignment * Handle entry/exit page visit props separately from props stored in events table * Update test which created sessions/events differently from everyone else * Make query_events private * Dont filter by session properties on events table if querying sessions and joining in events * Handle visits, pageviews, events and visitors metrics from other table * both -> either * events, pageviews are strictly event metrics * Add support for (plain) breakdowns deciding which table to use * Run tests with experimental_reduced_joins as a separate job Also refactor which tests are run with postgres:15 to reduce number of jobs * moduledocs for TableDecider * Fix matrix * Custom build name * Move TEST_EXPERIMENTAL_REDUCED_JOINS check * Handle percentage separately from other metrics * Remove debug code * TableDecider tests * both => sample_percent * Improve naming * Simplify code * Breakdowns retain old behavior if getting metric visitors * Unify behavior of entry/exit page hostnames with rest * Fix test namingpull/3899/merge r2024.04.04-425
parent
6af80dd246
commit
3115c6e7a8
@ -0,0 +1,111 @@
|
||||
defmodule Plausible.Stats.TableDecider do
|
||||
@moduledoc """
|
||||
This module contains logic for deciding which tables need to be queried given a query
|
||||
and metrics, with the purpose of reducing the number of queries and JOINs needed to perform.
|
||||
"""
|
||||
|
||||
import Enum, only: [empty?: 1]
|
||||
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def events_join_sessions?(query) do
|
||||
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
|
||||
end
|
||||
|
||||
def partition_metrics(metrics, query, breakdown_property \\ nil) do
|
||||
%{
|
||||
event: event_only_metrics,
|
||||
session: session_only_metrics,
|
||||
either: either_metrics,
|
||||
other: other_metrics,
|
||||
sample_percent: sample_percent
|
||||
} =
|
||||
partition(metrics, query, &metric_partitioner/2)
|
||||
|
||||
# Treat breakdown property as yet another filter
|
||||
filters =
|
||||
if breakdown_property do
|
||||
Map.put(query.filters, breakdown_property, nil)
|
||||
else
|
||||
query.filters
|
||||
end
|
||||
|
||||
%{event: event_only_filters, session: session_only_filters} =
|
||||
partition(filters, query, &filters_partitioner/2)
|
||||
|
||||
cond do
|
||||
# Only one table needs to be queried
|
||||
empty?(event_only_metrics) && empty?(event_only_filters) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) && empty?(session_only_filters) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Filters on both events and sessions, but only one kind of metric
|
||||
empty?(event_only_metrics) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Default: prefer sessions
|
||||
true ->
|
||||
{event_only_metrics ++ sample_percent,
|
||||
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
end
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, :conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :average_revenue), do: :event
|
||||
defp metric_partitioner(_, :total_revenue), do: :event
|
||||
defp metric_partitioner(_, :pageviews), do: :event
|
||||
defp metric_partitioner(_, :events), do: :event
|
||||
defp metric_partitioner(_, :bounce_rate), do: :session
|
||||
defp metric_partitioner(_, :visit_duration), do: :session
|
||||
defp metric_partitioner(_, :views_per_visit), do: :session
|
||||
|
||||
# Metrics which used to only be queried from one table but can be calculated from either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visits), do: :either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visitors), do: :either
|
||||
|
||||
defp metric_partitioner(_, :visits), do: :session
|
||||
defp metric_partitioner(_, :visitors), do: :event
|
||||
# Calculated metrics - handled on callsite separately from other metrics.
|
||||
defp metric_partitioner(_, :time_on_page), do: :other
|
||||
defp metric_partitioner(_, :total_visitors), do: :other
|
||||
defp metric_partitioner(_, :percentage), do: :other
|
||||
# Sample percentage is included in both tables if queried.
|
||||
defp metric_partitioner(_, :sample_percent), do: :sample_percent
|
||||
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
|
||||
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, _), do: :either
|
||||
|
||||
defp filters_partitioner(_, {"event:" <> _, _}), do: :event
|
||||
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
|
||||
do: :either
|
||||
|
||||
defp filters_partitioner(_, {"visit:" <> _, _}),
|
||||
do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
|
||||
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp filters_partitioner(_, _), do: :either
|
||||
|
||||
@default %{event: [], session: [], either: [], other: [], sample_percent: []}
|
||||
defp partition(values, query, partitioner) do
|
||||
Enum.reduce(values, @default, fn value, acc ->
|
||||
key = partitioner.(query, value)
|
||||
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,164 @@
|
||||
defmodule Plausible.Stats.TableDeciderTest do
|
||||
use Plausible.DataCase, async: true
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
import Plausible.Stats.TableDecider
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins disabled" do
|
||||
assert not events_join_sessions?(make_query(false, %{}))
|
||||
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
|
||||
assert events_join_sessions?(make_query(false, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins enabled" do
|
||||
assert not events_join_sessions?(make_query(true, %{}))
|
||||
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
|
||||
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
describe "partition_metrics" do
|
||||
test "with no metrics or filters" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([], query) == {[], [], []}
|
||||
end
|
||||
|
||||
test "session-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
|
||||
{[], [:bounce_rate, :views_per_visit], []}
|
||||
end
|
||||
|
||||
test "event-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, event-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, session-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
|
||||
end
|
||||
|
||||
test "session filters but no session metrics" do
|
||||
query = make_query(false, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "sample_percent is added to both types of metrics" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [], []}
|
||||
|
||||
assert partition_metrics([:bounce_rate, :sample_percent], query) ==
|
||||
{[], [:bounce_rate, :sample_percent], []}
|
||||
|
||||
assert partition_metrics([:total_revenue, :bounce_rate, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [:bounce_rate, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "other metrics put in its own result" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
|
||||
{[], [], [:time_on_page, :percentage, :total_visitors]}
|
||||
end
|
||||
|
||||
test "raises if unknown metric" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert_raise ArgumentError, fn ->
|
||||
partition_metrics([:foobar], query)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "partition_metrics with experimental_reduced_joins enabled" do
|
||||
test "metrics that can be calculated on either when event-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
|
||||
assert partition_metrics([:pageviews, :visits], query) == {[:pageviews, :visits], [], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either when session-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :visitors], query) ==
|
||||
{[], [:bounce_rate, :visitors], []}
|
||||
|
||||
assert partition_metrics([:visit_duration, :visits], query) ==
|
||||
{[], [:visit_duration, :visits], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either are biased to sessions" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
|
||||
{[:total_revenue], [:bounce_rate, :visitors], []}
|
||||
end
|
||||
|
||||
test "sample_percent is handled with either metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors, :sample_percent], query) ==
|
||||
{[], [:visitors, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events" do
|
||||
query = make_query(true, %{name: "pageview"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events and sessions" do
|
||||
query = make_query(true, %{name: "pageview", exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on either" do
|
||||
query = make_query(true, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on sessions" do
|
||||
query = make_query(true, %{exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "breakdown value leans metric" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
|
||||
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
|
||||
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
|
||||
end
|
||||
end
|
||||
|
||||
defp make_query(experimental_reduced_joins?, filters) do
|
||||
Query.from(build(:site), %{
|
||||
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
|
||||
"filters" => Jason.encode!(filters)
|
||||
})
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue