From d4d86637e40c3a58a9f22af4618fcccab1156592 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Thu, 18 Jan 2024 07:40:20 +0100 Subject: [PATCH] PoC: keep log_comment metadata in ClickHouse queries --- lib/plausible/clickhouse_repo.ex | 45 ++++++++++++++++++++++++++++++++ lib/plausible/stats.ex | 6 +++++ lib/plausible/stats/query.ex | 2 ++ lib/plausible_web/router.ex | 4 +++ 4 files changed, 57 insertions(+) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index f6f8631d9..fff178219 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -12,6 +12,51 @@ defmodule Plausible.ClickhouseRepo do end end + defmodule KeepRouteContext do + alias Plausible.ClickhouseRepo + + def init(_opts), do: nil + + def call(conn, _) do + ClickhouseRepo.set_context(%{request_path: conn.request_path}) + conn + end + end + + @logger_metadata_key __MODULE__ + def set_context(new) when is_map(new) do + metadata = + case :logger.get_process_metadata() do + %{@logger_metadata_key => ctx} -> Map.update(ctx, :log_comment, new, &Map.merge(&1, new)) + _ -> %{:log_comment => new} + end + + :logger.update_process_metadata(%{@logger_metadata_key => metadata}) + end + + def get_context() do + case :logger.get_process_metadata() do + %{@logger_metadata_key => ctx} -> + ctx + + %{} -> + %{} + + :undefined -> + %{} + end + end + + @impl true + def prepare_query(_operation, query, opts) do + setting = {:log_comment, Jason.encode!(get_context())} + + opts = + Keyword.update(opts, :settings, [setting], fn settings -> [setting | settings] end) + + {query, opts} + end + @task_timeout 60_000 def parallel_tasks(queries) do ctx = OpenTelemetry.Ctx.get_current() diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex index b37e8fe77..d45461200 100644 --- a/lib/plausible/stats.ex +++ b/lib/plausible/stats.ex @@ -13,32 +13,38 @@ defmodule Plausible.Stats do def breakdown(site, query, prop, metrics, pagination) do include_sentry_replay_info() + Plausible.ClickhouseRepo.set_context(%{query_type: :breakdown}) Breakdown.breakdown(site, query, prop, metrics, pagination) end def aggregate(site, query, metrics) do + Plausible.ClickhouseRepo.set_context(%{query_type: :aggregate}) include_sentry_replay_info() Aggregate.aggregate(site, query, metrics) end def timeseries(site, query, metrics) do + Plausible.ClickhouseRepo.set_context(%{query_type: :timeseries}) include_sentry_replay_info() Timeseries.timeseries(site, query, metrics) end def current_visitors(site) do + Plausible.ClickhouseRepo.set_context(%{query_type: :current_visitors}) include_sentry_replay_info() CurrentVisitors.current_visitors(site) end on_full_build do def funnel(site, query, funnel) do + Plausible.ClickhouseRepo.set_context(%{query_type: :funnel}) include_sentry_replay_info() Plausible.Stats.Funnel.funnel(site, query, funnel) end end def filter_suggestions(site, query, filter_name, filter_search) do + Plausible.ClickhouseRepo.set_context(%{query_type: :filter_suggestions}) include_sentry_replay_info() FilterSuggestions.filter_suggestions(site, query, filter_name, filter_search) end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index d00b27eaa..1a436dfb4 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -15,6 +15,8 @@ defmodule Plausible.Stats.Query do @type t :: %__MODULE__{} def from(site, params) do + Plausible.ClickhouseRepo.set_context(params) + query = query_by_period(site, params) |> put_interval(params) diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index cebf2927f..df75429f1 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -14,6 +14,7 @@ defmodule PlausibleWeb.Router do plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_seconds plug PlausibleWeb.AuthPlug plug PlausibleWeb.LastSeenPlug + plug Plausible.ClickhouseRepo.KeepRouteContext end pipeline :shared_link do @@ -38,6 +39,7 @@ defmodule PlausibleWeb.Router do plug :accepts, ["json"] plug :fetch_session plug PlausibleWeb.AuthPlug + plug Plausible.ClickhouseRepo.KeepRouteContext end pipeline :internal_stats_api do @@ -45,10 +47,12 @@ defmodule PlausibleWeb.Router do plug :fetch_session plug PlausibleWeb.AuthorizeSiteAccess plug PlausibleWeb.Plugs.NoRobots + plug Plausible.ClickhouseRepo.KeepRouteContext end pipeline :public_api do plug :accepts, ["json"] + plug Plausible.ClickhouseRepo.KeepRouteContext end on_full_build do