Move sampling outside of small build

pull/3559/head
Adam Rutkowski 3 years ago
parent 62790de74b
commit 21aaa7e290

@ -0,0 +1,41 @@
defmodule Plausible.Stats.Sampling do
@moduledoc """
Sampling related functions
"""
@default_sample_threshold 20_000_000
import Ecto.Query
@spec add_query_hint(Ecto.Query.t(), Plausible.Query.t()) :: Ecto.Query.t()
def add_query_hint(%Ecto.Query{} = db_query, %Plausible.Stats.Query{} = query) do
case query.sample_threshold do
:infinite ->
db_query
threshold ->
add_query_hint(db_query, threshold)
end
end
@spec add_query_hint(Ecto.Query.t(), pos_integer()) :: Ecto.Query.t()
def add_query_hint(%Ecto.Query{} = query, threshold) when is_integer(threshold) do
from(x in query, hints: [sample: threshold])
end
@spec add_query_hint(Ecto.Query.t()) :: Ecto.Query.t()
def add_query_hint(%Ecto.Query{} = query) do
add_query_hint(query, @default_sample_threshold)
end
@spec put_threshold(Plausible.Query.t(), map()) :: Plausible.Query.t()
def put_threshold(query, params) do
sample_threshold =
case params["sample_threshold"] do
nil -> @default_sample_threshold
"infinite" -> :infinite
value -> String.to_integer(value)
end
Map.put(query, :sample_threshold, sample_threshold)
end
end

@ -4,8 +4,6 @@ defmodule Plausible.Stats.Base do
alias Plausible.Stats.{Query, Filters}
import Ecto.Query
# Ecto typespec has not been updated for this PR: https://github.com/elixir-ecto/ecto/pull/3592
@dialyzer {:nowarn_function, add_sample_hint: 2}
@no_ref "Direct / None"
@not_set "(not set)"
@ -40,7 +38,10 @@ defmodule Plausible.Stats.Base do
where: e.site_id == ^site.id,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
|> add_sample_hint(query)
on_full_build do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname))
@ -211,8 +212,12 @@ defmodule Plausible.Stats.Base do
where: s.site_id == ^site.id,
where: s.start >= ^first_datetime and s.start < ^last_datetime
)
|> add_sample_hint(query)
|> filter_by_entry_props(query)
on_full_build do
sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
end
sessions_q = filter_by_entry_props(sessions_q, query)
Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->
filter_key = "visit:" <> prop_name
@ -549,16 +554,6 @@ defmodule Plausible.Stats.Base do
end)
end
defp add_sample_hint(db_q, query) do
case query.sample_threshold do
:infinite ->
db_q
threshold ->
from(e in db_q, hints: [sample: threshold])
end
end
defp split_goals(clauses, map_fn \\ &Function.identity/1) do
groups =
Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)

@ -1,4 +1,5 @@
defmodule Plausible.Stats.Clickhouse do
use Plausible
use Plausible.Repo
use Plausible.ClickhouseRepo
use Plausible.Stats.Fragments
@ -236,7 +237,6 @@ defmodule Plausible.Stats.Clickhouse do
current_q =
from(
e in "events_v2",
hints: [sample: 20_000_000],
join: total_q in subquery(total_q),
on: e.site_id == total_q.site_id,
where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
@ -252,6 +252,10 @@ defmodule Plausible.Stats.Clickhouse do
order_by: [e.site_id, fragment("toStartOfHour(timestamp)")]
)
on_full_build do
current_q = Plausible.Stats.Sampling.add_query_hint(current_q)
end
result =
current_q
|> ClickhouseRepo.all()
@ -275,17 +279,22 @@ defmodule Plausible.Stats.Clickhouse do
end
defp visitors_24h_total(now, offset1, offset2, site_id_to_domain_mapping) do
from(e in "events_v2",
hints: [sample: 20_000_000],
where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour),
where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour),
select: %{
site_id: e.site_id,
total_visitors: fragment("toUInt64(round(uniq(user_id) * any(_sample_factor)))")
},
group_by: [e.site_id]
)
query =
from e in "events_v2",
where: e.site_id in ^Map.keys(site_id_to_domain_mapping),
where: e.timestamp >= ^NaiveDateTime.add(now, offset1, :hour),
where: e.timestamp <= ^NaiveDateTime.add(now, offset2, :hour),
select: %{
site_id: e.site_id,
total_visitors: fragment("toUInt64(round(uniq(user_id) * any(_sample_factor)))")
},
group_by: [e.site_id]
on_full_build do
query = Plausible.Stats.Sampling.add_query_hint(query)
end
query
end
defp empty_24h_intervals(now) do
@ -306,11 +315,14 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(s in "sessions_v2",
hints: ["SAMPLE 10000000"],
where: s.site_id == ^site.id,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
)
on_full_build do
q = Plausible.Stats.Sampling.add_query_hint(q, 10_000_000)
end
q =
if query.filters["source"] do
source = query.filters["source"]
@ -425,11 +437,14 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(e in "events_v2",
hints: ["SAMPLE 10000000"],
where: e.site_id == ^site.id,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
on_full_build do
q = Plausible.Stats.Sampling.add_query_hint(q, 10_000_000)
end
q =
if query.filters["screen"] do
size = query.filters["screen"]

@ -1,4 +1,6 @@
defmodule Plausible.Stats.Query do
use Plausible
defstruct date_range: nil,
interval: nil,
period: nil,
@ -7,19 +9,24 @@ defmodule Plausible.Stats.Query do
imported_data_requested: false,
include_imported: false
@default_sample_threshold 20_000_000
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{FilterParser, Interval}
@type t :: %__MODULE__{}
def from(site, params) do
query_by_period(site, params)
|> put_interval(params)
|> put_parsed_filters(params)
|> put_imported_opts(site, params)
|> put_sample_threshold(params)
|> maybe_drop_prop_filter(site)
query =
query_by_period(site, params)
|> put_interval(params)
|> put_parsed_filters(params)
|> put_imported_opts(site, params)
|> maybe_drop_prop_filter(site)
on_full_build do
query = Plausible.Stats.Sampling.put_threshold(query, params)
end
query
end
defp query_by_period(site, %{"period" => "realtime"}) do
@ -163,17 +170,6 @@ defmodule Plausible.Stats.Query do
Map.put(query, :filters, FilterParser.parse_filters(params["filters"]))
end
defp put_sample_threshold(query, params) do
sample_threshold =
case params["sample_threshold"] do
nil -> @default_sample_threshold
"infinite" -> :infinite
value -> String.to_integer(value)
end
Map.put(query, :sample_threshold, sample_threshold)
end
def put_filter(query, key, val) do
%__MODULE__{
query

@ -158,11 +158,13 @@ defmodule Plausible.Stats.QueryTest do
assert q.interval == "date"
end
@tag :full_build_only
test "adds sample_threshold :infinite to query struct", %{site: site} do
q = Query.from(site, %{"period" => "30d", "sample_threshold" => "infinite"})
assert q.sample_threshold == :infinite
end
@tag :full_build_only
test "casts sample_threshold to integer in query struct", %{site: site} do
q = Query.from(site, %{"period" => "30d", "sample_threshold" => "30000000"})
assert q.sample_threshold == 30_000_000

Loading…
Cancel
Save