diff --git a/extra/lib/plausible/stats/sampling.ex b/extra/lib/plausible/stats/sampling.ex new file mode 100644 index 000000000..c72b7bac6 --- /dev/null +++ b/extra/lib/plausible/stats/sampling.ex @@ -0,0 +1,41 @@ +defmodule Plausible.Stats.Sampling do + @moduledoc """ + Sampling related functions + """ + @default_sample_threshold 20_000_000 + + import Ecto.Query + + @spec add_query_hint(Ecto.Query.t(), Plausible.Query.t()) :: Ecto.Query.t() + def add_query_hint(%Ecto.Query{} = db_query, %Plausible.Stats.Query{} = query) do + case query.sample_threshold do + :infinite -> + db_query + + threshold -> + add_query_hint(db_query, threshold) + end + end + + @spec add_query_hint(Ecto.Query.t(), pos_integer()) :: Ecto.Query.t() + def add_query_hint(%Ecto.Query{} = query, threshold) when is_integer(threshold) do + from(x in query, hints: [sample: threshold]) + end + + @spec add_query_hint(Ecto.Query.t()) :: Ecto.Query.t() + def add_query_hint(%Ecto.Query{} = query) do + add_query_hint(query, @default_sample_threshold) + end + + @spec put_threshold(Plausible.Query.t(), map()) :: Plausible.Query.t() + def put_threshold(query, params) do + sample_threshold = + case params["sample_threshold"] do + nil -> @default_sample_threshold + "infinite" -> :infinite + value -> String.to_integer(value) + end + + Map.put(query, :sample_threshold, sample_threshold) + end +end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index ecdb922ff..684d2676f 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -4,8 +4,6 @@ defmodule Plausible.Stats.Base do alias Plausible.Stats.{Query, Filters} import Ecto.Query - # Ecto typespec has not been updated for this PR: https://github.com/elixir-ecto/ecto/pull/3592 - @dialyzer {:nowarn_function, add_sample_hint: 2} @no_ref "Direct / None" @not_set "(not set)" @@ -40,7 +38,10 @@ defmodule Plausible.Stats.Base do where: e.site_id == ^site.id, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) - |> add_sample_hint(query) + + on_full_build do + q = Plausible.Stats.Sampling.add_query_hint(q, query) + end q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname)) @@ -211,8 +212,12 @@ defmodule Plausible.Stats.Base do where: s.site_id == ^site.id, where: s.start >= ^first_datetime and s.start < ^last_datetime ) - |> add_sample_hint(query) - |> filter_by_entry_props(query) + + on_full_build do + sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query) + end + + sessions_q = filter_by_entry_props(sessions_q, query) Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q -> filter_key = "visit:" <> prop_name @@ -549,16 +554,6 @@ defmodule Plausible.Stats.Base do end) end - defp add_sample_hint(db_q, query) do - case query.sample_threshold do - :infinite -> - db_q - - threshold -> - from(e in db_q, hints: [sample: threshold]) - end - end - defp split_goals(clauses, map_fn \\ &Function.identity/1) do groups = Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end) diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index adefd1822..909f71f56 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -1,4 +1,5 @@ defmodule Plausible.Stats.Clickhouse do + use Plausible use Plausible.Repo use Plausible.ClickhouseRepo use Plausible.Stats.Fragments @@ -236,7 +237,6 @@ defmodule Plausible.Stats.Clickhouse do current_q = from( e in "events_v2", - hints: [sample: 20_000_000], join: total_q in subquery(total_q), on: e.site_id == total_q.site_id, where: e.site_id in ^Map.keys(site_id_to_domain_mapping), @@ -252,6 +252,10 @@ defmodule Plausible.Stats.Clickhouse do order_by: [e.site_id, fragment("toStartOfHour(timestamp)")] ) + on_full_build do + current_q = Plausible.Stats.Sampling.add_query_hint(current_q) + end + result = current_q |> ClickhouseRepo.all() @@ -275,17 +279,22 @@ defmodule Plausible.Stats.Clickhouse do end defp visitors_24h_total(now, offset1, offset2, site_id_to_domain_mapping) do - from(e in "events_v2", - hints: [sample: 20_000_000], - where: e.site_id in ^Map.keys(site_id_to_domain_mapping), - where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour), - where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour), - select: %{ - site_id: e.site_id, - total_visitors: fragment("toUInt64(round(uniq(user_id) * any(_sample_factor)))") - }, - group_by: [e.site_id] - ) + query = + from e in "events_v2", + where: e.site_id in ^Map.keys(site_id_to_domain_mapping), + where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour), + where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour), + select: %{ + site_id: e.site_id, + total_visitors: fragment("toUInt64(round(uniq(user_id) * any(_sample_factor)))") + }, + group_by: [e.site_id] + + on_full_build do + query = Plausible.Stats.Sampling.add_query_hint(query) + end + + query end defp empty_24h_intervals(now) do @@ -306,11 +315,14 @@ defmodule Plausible.Stats.Clickhouse do q = from(s in "sessions_v2", - hints: ["SAMPLE 10000000"], where: s.site_id == ^site.id, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime ) + on_full_build do + q = Plausible.Stats.Sampling.add_query_hint(q, 10_000_000) + end + q = if query.filters["source"] do source = query.filters["source"] @@ -425,11 +437,14 @@ defmodule Plausible.Stats.Clickhouse do q = from(e in "events_v2", - hints: ["SAMPLE 10000000"], where: e.site_id == ^site.id, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) + on_full_build do + q = Plausible.Stats.Sampling.add_query_hint(q, 10_000_000) + end + q = if query.filters["screen"] do size = query.filters["screen"] diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 6c52f79e9..d00b27eaa 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,4 +1,6 @@ defmodule Plausible.Stats.Query do + use Plausible + defstruct date_range: nil, interval: nil, period: nil, @@ -7,19 +9,24 @@ defmodule Plausible.Stats.Query do imported_data_requested: false, include_imported: false - @default_sample_threshold 20_000_000 require OpenTelemetry.Tracer, as: Tracer alias Plausible.Stats.{FilterParser, Interval} @type t :: %__MODULE__{} def from(site, params) do - query_by_period(site, params) - |> put_interval(params) - |> put_parsed_filters(params) - |> put_imported_opts(site, params) - |> put_sample_threshold(params) - |> maybe_drop_prop_filter(site) + query = + query_by_period(site, params) + |> put_interval(params) + |> put_parsed_filters(params) + |> put_imported_opts(site, params) + |> maybe_drop_prop_filter(site) + + on_full_build do + query = Plausible.Stats.Sampling.put_threshold(query, params) + end + + query end defp query_by_period(site, %{"period" => "realtime"}) do @@ -163,17 +170,6 @@ defmodule Plausible.Stats.Query do Map.put(query, :filters, FilterParser.parse_filters(params["filters"])) end - defp put_sample_threshold(query, params) do - sample_threshold = - case params["sample_threshold"] do - nil -> @default_sample_threshold - "infinite" -> :infinite - value -> String.to_integer(value) - end - - Map.put(query, :sample_threshold, sample_threshold) - end - def put_filter(query, key, val) do %__MODULE__{ query diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 415f66e67..49b4033c2 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -158,11 +158,13 @@ defmodule Plausible.Stats.QueryTest do assert q.interval == "date" end + @tag :full_build_only test "adds sample_threshold :infinite to query struct", %{site: site} do q = Query.from(site, %{"period" => "30d", "sample_threshold" => "infinite"}) assert q.sample_threshold == :infinite end + @tag :full_build_only test "casts sample_threshold to integer in query struct", %{site: site} do q = Query.from(site, %{"period" => "30d", "sample_threshold" => "30000000"}) assert q.sample_threshold == 30_000_000