From 960e15e25bc87333cbcfe0b3d23ed3d20fc05c89 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 12 Mar 2024 13:57:30 +0200 Subject: [PATCH] WIP new querying system for deciding what tables to query --- lib/plausible/stats/aggregate.ex | 21 ++------ lib/plausible/stats/query.ex | 16 +++++- lib/plausible/stats/table_decider.ex | 77 ++++++++++++++++++++++++++++ lib/plausible/stats/timeseries.ex | 13 ++--- 4 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 lib/plausible/stats/table_decider.ex diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 68b96183a..a4aec17bb 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -5,19 +5,6 @@ defmodule Plausible.Stats.Aggregate do import Ecto.Query alias Plausible.Stats.{Query, Util} - @revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - - @event_metrics [ - :visitors, - :pageviews, - :events, - :sample_percent, - :conversion_rate, - :total_visitors - ] ++ @revenue_metrics - - @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent] - def aggregate(site, query, metrics) do {currency, metrics} = on_full_build do @@ -28,18 +15,18 @@ defmodule Plausible.Stats.Aggregate do Query.trace(query, metrics) - event_metrics = + {event_metrics, session_metrics, other_metrics} = metrics |> Util.maybe_add_visitors_metric() - |> Enum.filter(&(&1 in @event_metrics)) + |> Plausible.Stats.TableDecider.partition_metrics(query) event_task = fn -> aggregate_events(site, query, event_metrics) end - session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) + # session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) session_task = fn -> aggregate_sessions(site, query, session_metrics) end time_on_page_task = - if :time_on_page in metrics do + if :time_on_page in other_metrics do fn -> aggregate_time_on_page(site, query) end else fn -> %{} end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 6874aaa96..852f4e372 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -9,7 +9,8 @@ defmodule Plausible.Stats.Query do imported_data_requested: false, include_imported: false, now: nil, - experimental_session_count?: false + experimental_session_count?: false, + experimental_reduced_joins?: false require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{Filters, Interval} @@ -23,6 +24,7 @@ defmodule Plausible.Stats.Query do __MODULE__ |> struct!(now: now) |> put_experimental_session_count(site, params) + |> put_experimental_reduced_joins(site, params) |> put_period(site, params) |> put_interval(params) |> put_parsed_filters(params) @@ -48,6 +50,18 @@ defmodule Plausible.Stats.Query do end end + defp put_experimental_reduced_joins(query, site, params) do + if Map.has_key?(params, "experimental_reduced_joins") do + struct!(query, + experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true" + ) + else + struct!(query, + experimental_reduced_joins?: FunWithFlags.enabled?(:experimental_reduced_joins, for: site) + ) + end + end + defp put_period(query, site, %{"period" => "realtime"}) do date = today(site.timezone) diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex new file mode 100644 index 000000000..0ee37b4d4 --- /dev/null +++ b/lib/plausible/stats/table_decider.ex @@ -0,0 +1,77 @@ +defmodule Plausible.Stats.TableDecider do + import Enum, only: [empty?: 1] + + alias Plausible.Stats.Query + + def partition_metrics(metrics, query) do + %{event: event_metrics, session: session_metrics, both: both_metrics, other: other_metrics} = + partition(metrics, query, &metric_partitioner/2) + + %{event: event_filters, session: session_filters, both: _both_filters} = + partition(query.filters, query, &filters_partitioner/2) + + cond do + # Only one table needs to be queried + empty?(session_metrics) && empty?(session_filters) -> + {event_metrics ++ both_metrics, [], other_metrics} + + empty?(event_metrics) && empty?(event_filters) -> + {[], session_metrics ++ both_metrics, other_metrics} + + # Filters from both, but only one kind of metric + empty?(session_metrics) -> + {event_metrics ++ both_metrics, [], other_metrics} + + # Default: prefer sessions + true -> + {event_metrics, session_metrics ++ both_metrics, other_metrics} + end + end + + defp metric_partitioner(_, :conversion_rate), do: :event + defp metric_partitioner(_, :average_revenue), do: :event + defp metric_partitioner(_, :total_revenue), do: :event + defp metric_partitioner(_, :bounce_rate), do: :session + defp metric_partitioner(_, :visit_duration), do: :session + defp metric_partitioner(_, :views_per_visit), do: :session + # :TODO: These can be calculated on the other table as well! + defp metric_partitioner(_, :visits), do: :session + defp metric_partitioner(_, :pageviews), do: :event + defp metric_partitioner(_, :events), do: :event + defp metric_partitioner(_, :visitors), do: :event + # :TODO: Calculated/weird metrics + defp metric_partitioner(_, :time_on_page), do: :other + defp metric_partitioner(_, :total_visitors), do: :other + # :TODO: Should be included on _both_ + defp metric_partitioner(_, :sample_percent), do: :event + + defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do + raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?" + end + + defp metric_partitioner(_, _), do: :both + + defp filters_partitioner(_, {"event:name", _}), do: :event + defp filters_partitioner(_, {"event:page", _}), do: :event + defp filters_partitioner(_, {"event:goal", _}), do: :event + defp filters_partitioner(_, {"event:props:" <> _prop, _}), do: :event + defp filters_partitioner(_, {"visit:entry_page", _}), do: :session + defp filters_partitioner(_, {"visit:exit_page", _}), do: :session + + defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {"visit:" <> _, _}), + do: :session + + defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do + raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?" + end + + defp filters_partitioner(_, _), do: :both + + @default %{event: [], session: [], both: [], other: []} + defp partition(values, query, partitioner) do + Enum.reduce(values, @default, fn value, acc -> + key = partitioner.(query, value) + Map.put(acc, key, Map.fetch!(acc, key) ++ [value]) + end) + end +end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 2efa6418b..866127bcb 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -18,15 +18,16 @@ defmodule Plausible.Stats.Timeseries do @typep value :: nil | integer() | float() @type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()}) - @revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) - - @event_metrics [:visitors, :pageviews, :events, :conversion_rate] ++ @revenue_metrics - @session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit] def timeseries(site, query, metrics) do steps = buckets(query) - event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) - session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) + # event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) + # session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) + + {event_metrics, session_metrics, _} = + Plausible.Stats.TableDecider.partition_metrics(metrics, query) + + # IO.inspect(t: "timeseries", event_metrics: event_metrics, session_metrics: session_metrics) {currency, event_metrics} = on_full_build do