WIP new querying system for deciding what tables to query

pull/3966/head
Karl-Aksel Puulmann 3 years ago
parent 08211661cb
commit 960e15e25b

@ -5,19 +5,6 @@ defmodule Plausible.Stats.Aggregate do
import Ecto.Query
alias Plausible.Stats.{Query, Util}
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [
:visitors,
:pageviews,
:events,
:sample_percent,
:conversion_rate,
:total_visitors
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
def aggregate(site, query, metrics) do
{currency, metrics} =
on_full_build do
@ -28,18 +15,18 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics)
event_metrics =
{event_metrics, session_metrics, other_metrics} =
metrics
|> Util.maybe_add_visitors_metric()
|> Enum.filter(&(&1 in @event_metrics))
|> Plausible.Stats.TableDecider.partition_metrics(query)
event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
# session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task =
if :time_on_page in metrics do
if :time_on_page in other_metrics do
fn -> aggregate_time_on_page(site, query) end
else
fn -> %{} end

@ -9,7 +9,8 @@ defmodule Plausible.Stats.Query do
imported_data_requested: false,
include_imported: false,
now: nil,
experimental_session_count?: false
experimental_session_count?: false,
experimental_reduced_joins?: false
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval}
@ -23,6 +24,7 @@ defmodule Plausible.Stats.Query do
__MODULE__
|> struct!(now: now)
|> put_experimental_session_count(site, params)
|> put_experimental_reduced_joins(site, params)
|> put_period(site, params)
|> put_interval(params)
|> put_parsed_filters(params)
@ -48,6 +50,18 @@ defmodule Plausible.Stats.Query do
end
end
defp put_experimental_reduced_joins(query, site, params) do
if Map.has_key?(params, "experimental_reduced_joins") do
struct!(query,
experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"
)
else
struct!(query,
experimental_reduced_joins?: FunWithFlags.enabled?(:experimental_reduced_joins, for: site)
)
end
end
defp put_period(query, site, %{"period" => "realtime"}) do
date = today(site.timezone)

@ -0,0 +1,77 @@
defmodule Plausible.Stats.TableDecider do
import Enum, only: [empty?: 1]
alias Plausible.Stats.Query
def partition_metrics(metrics, query) do
%{event: event_metrics, session: session_metrics, both: both_metrics, other: other_metrics} =
partition(metrics, query, &metric_partitioner/2)
%{event: event_filters, session: session_filters, both: _both_filters} =
partition(query.filters, query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
empty?(session_metrics) && empty?(session_filters) ->
{event_metrics ++ both_metrics, [], other_metrics}
empty?(event_metrics) && empty?(event_filters) ->
{[], session_metrics ++ both_metrics, other_metrics}
# Filters from both, but only one kind of metric
empty?(session_metrics) ->
{event_metrics ++ both_metrics, [], other_metrics}
# Default: prefer sessions
true ->
{event_metrics, session_metrics ++ both_metrics, other_metrics}
end
end
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
defp metric_partitioner(_, :total_revenue), do: :event
defp metric_partitioner(_, :bounce_rate), do: :session
defp metric_partitioner(_, :visit_duration), do: :session
defp metric_partitioner(_, :views_per_visit), do: :session
# :TODO: These can be calculated on the other table as well!
defp metric_partitioner(_, :visits), do: :session
defp metric_partitioner(_, :pageviews), do: :event
defp metric_partitioner(_, :events), do: :event
defp metric_partitioner(_, :visitors), do: :event
# :TODO: Calculated/weird metrics
defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other
# :TODO: Should be included on _both_
defp metric_partitioner(_, :sample_percent), do: :event
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
end
defp metric_partitioner(_, _), do: :both
defp filters_partitioner(_, {"event:name", _}), do: :event
defp filters_partitioner(_, {"event:page", _}), do: :event
defp filters_partitioner(_, {"event:goal", _}), do: :event
defp filters_partitioner(_, {"event:props:" <> _prop, _}), do: :event
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {"visit:" <> _, _}),
do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
end
defp filters_partitioner(_, _), do: :both
@default %{event: [], session: [], both: [], other: []}
defp partition(values, query, partitioner) do
Enum.reduce(values, @default, fn value, acc ->
key = partitioner.(query, value)
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
end)
end
end

@ -18,15 +18,16 @@ defmodule Plausible.Stats.Timeseries do
@typep value :: nil | integer() | float()
@type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()})
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [:visitors, :pageviews, :events, :conversion_rate] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit]
def timeseries(site, query, metrics) do
steps = buckets(query)
event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
# event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
# session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
{event_metrics, session_metrics, _} =
Plausible.Stats.TableDecider.partition_metrics(metrics, query)
# IO.inspect(t: "timeseries", event_metrics: event_metrics, session_metrics: session_metrics)
{currency, event_metrics} =
on_full_build do

Loading…
Cancel
Save