From 356575ef78da26cf8dfaa3fb1d05ace24d7f5442 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Mon, 28 Nov 2022 15:50:55 +0100 Subject: [PATCH] Gatekeep ingestion pipeline (#2472) * Update Sites.Cache So it's now capable of refreshing most recent sites. Refreshing a single site is no longer wanted. * Introduce Warmer.RecentlyUpdated This is Sites Cache warmer that runs only for most recently updated sites every 30s. * Validate Request creation early * Rename RateLimiter to GateKeeper and introduce detailed policies * Update events API tests - a provisioned site is now required * Update events ingestion tests * Make limits visible in CRM Sites index * Hard-deprecate DOMAIN_BLACKLIST * Remove unnecessary clause * Fix typo * Explicitly delegate Warmer.All * GateKeeper.allwoance => GateKeeper.check * Instrument Sites.Cache measurments * Update send_pageview task to output response headers * Instrument ingestion pipeline * Credo * Make event telemetry test a sync case * Simplify Request.uri/hostname handling * Use embedded schema, apply action and rely on get_field --- config/runtime.exs | 8 +- lib/mix/tasks/send_pageview.ex | 4 +- lib/plausible/application.ex | 7 +- lib/plausible/ingestion/event.ex | 98 ++-- lib/plausible/ingestion/request.ex | 212 +++++--- lib/plausible/site/admin.ex | 11 +- lib/plausible/site/cache.ex | 139 +++-- lib/plausible/site/cache/rate_limiter.ex | 109 ---- lib/plausible/site/cache/warmer.ex | 6 +- lib/plausible/site/cache/warmer/all.ex | 8 + .../site/cache/warmer/recently_updated.ex | 20 + lib/plausible/site/gate_keeper.ex | 87 ++++ lib/plausible/telemetry/plausible_metrics.ex | 58 ++- .../controllers/api/external_controller.ex | 4 +- .../ingestion/event_telemetry_test.exs | 44 ++ test/plausible/ingestion/event_test.exs | 206 ++++---- test/plausible/ingestion/request_test.exs | 187 +++++++ test/plausible/site/cache_test.exs | 60 +-- ..._limiter_test.exs => gate_keeper_test.exs} | 79 +-- .../api/external_controller_test.exs | 475 ++++++++++-------- 20 files changed, 1069 insertions(+), 753 deletions(-) delete mode 100644 lib/plausible/site/cache/rate_limiter.ex create mode 100644 lib/plausible/site/cache/warmer/all.ex create mode 100644 lib/plausible/site/cache/warmer/recently_updated.ex create mode 100644 lib/plausible/site/gate_keeper.ex create mode 100644 test/plausible/ingestion/event_telemetry_test.exs create mode 100644 test/plausible/ingestion/request_test.exs rename test/plausible/site/{rate_limiter_test.exs => gate_keeper_test.exs} (60%) diff --git a/config/runtime.exs b/config/runtime.exs index 08e14b159..579c53258 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -158,11 +158,6 @@ log_level = |> get_var_from_path_or_env("LOG_LEVEL", "warn") |> String.to_existing_atom() -domain_blacklist = - config_dir - |> get_var_from_path_or_env("DOMAIN_BLACKLIST", "") - |> String.split(",") - is_selfhost = config_dir |> get_var_from_path_or_env("SELFHOST", "true") @@ -195,8 +190,7 @@ config :plausible, site_limit: site_limit, site_limit_exempt: site_limit_exempt, is_selfhost: is_selfhost, - custom_script_name: custom_script_name, - domain_blacklist: domain_blacklist + custom_script_name: custom_script_name config :plausible, :selfhost, enable_email_verification: enable_email_verification, diff --git a/lib/mix/tasks/send_pageview.ex b/lib/mix/tasks/send_pageview.ex index 14840a5d3..29b69bec1 100644 --- a/lib/mix/tasks/send_pageview.ex +++ b/lib/mix/tasks/send_pageview.ex @@ -50,11 +50,13 @@ defmodule Mix.Tasks.SendPageview do body = get_body(parsed_opts) case Plausible.HTTPClient.post(url, headers, body) do - {:ok, _} -> + {:ok, resp} -> IO.puts( "✅ Successfully sent #{body[:name]} event to #{url} ✅ \n\nip=#{ip}\nuser_agent=#{user_agent}\nbody= #{inspect(body, pretty: true)}" ) + IO.puts("Response headers: " <> inspect(resp.headers, pretty: true)) + {:error, e} -> IO.puts("❌ Could not send event to #{url}. Got the following error: \n\n #{inspect(e)}") end diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index b060c3449..d9d015a84 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -21,11 +21,12 @@ defmodule Plausible.Application do Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true}, id: :cachex_sessions ), + Plausible.PromEx, {Plausible.Site.Cache, []}, - {Plausible.Site.Cache.Warmer, []}, + {Plausible.Site.Cache.Warmer.All, []}, + {Plausible.Site.Cache.Warmer.RecentlyUpdated, []}, PlausibleWeb.Endpoint, - {Oban, Application.get_env(:plausible, Oban)}, - Plausible.PromEx + {Oban, Application.get_env(:plausible, Oban)} ] opts = [strategy: :one_for_one, name: Plausible.Supervisor] diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 867cb331d..f4c4cfb92 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -7,6 +7,7 @@ defmodule Plausible.Ingestion.Event do """ alias Plausible.Ingestion.{Request, CityOverrides} alias Plausible.ClickhouseEvent + alias Plausible.Site.GateKeeper defstruct domain: nil, clickhouse_event_attrs: %{}, @@ -14,13 +15,14 @@ defmodule Plausible.Ingestion.Event do dropped?: false, drop_reason: nil, request: nil, - salts: nil + salts: nil, + changeset: nil @type drop_reason() :: :bot - | :domain_blocked | :spam_referrer - | {:error, Ecto.Changeset.t()} + | GateKeeper.policy() + | :invalid @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -29,26 +31,28 @@ defmodule Plausible.Ingestion.Event do dropped?: boolean(), drop_reason: drop_reason(), request: Request.t(), - salts: map() + salts: map(), + changeset: %Ecto.Changeset{} } - @spec build_and_buffer(Request.t()) :: - {:ok, %{dropped: [t()], buffered: [t()]}} + @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} def build_and_buffer(%Request{domains: domains} = request) do processed_events = if spam_referrer?(request) do for domain <- domains, do: drop(new(domain, request), :spam_referrer) else Enum.reduce(domains, [], fn domain, acc -> - if domain_blocked?(domain) do - [drop(new(domain, request), :domain_blocked) | acc] - else - processed = - domain - |> new(request) - |> process_unless_dropped(pipeline()) - - [processed | acc] + case GateKeeper.check(domain) do + :allow -> + processed = + domain + |> new(request) + |> process_unless_dropped(pipeline()) + + [processed | acc] + + {:deny, reason} -> + [drop(new(domain, request), reason) | acc] end end) end @@ -57,6 +61,16 @@ defmodule Plausible.Ingestion.Event do {:ok, %{dropped: dropped, buffered: buffered}} end + @spec telemetry_event_buffered() :: [atom()] + def telemetry_event_buffered() do + [:plausible, :ingest, :event, :buffered] + end + + @spec telemetry_event_dropped() :: [atom()] + def telemetry_event_dropped() do + [:plausible, :ingest, :event, :dropped] + end + defp pipeline() do [ &put_user_agent/1, @@ -84,15 +98,21 @@ defmodule Plausible.Ingestion.Event do end defp new(domain, request) do - %__MODULE__{domain: domain, request: request} + struct!(__MODULE__, domain: domain, request: request) end - defp drop(%__MODULE__{} = event, reason) do - %{event | dropped?: true, drop_reason: reason} + defp drop(%__MODULE__{} = event, reason, attrs \\ []) do + fields = + attrs + |> Keyword.put(:dropped?, true) + |> Keyword.put(:drop_reason, reason) + + emit_telemetry_dropped(reason) + struct!(event, fields) end defp update_attrs(%__MODULE__{} = event, %{} = attrs) do - %{event | clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)} + struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) end defp put_user_agent(%__MODULE__{} = event) do @@ -117,19 +137,12 @@ defmodule Plausible.Ingestion.Event do end defp put_basic_info(%__MODULE__{} = event) do - host = - case event.request.uri do - %{host: ""} -> "(none)" - %{host: host} when is_binary(host) -> host - _ -> nil - end - update_attrs(event, %{ domain: event.domain, - timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), + timestamp: event.request.timestamp, name: event.request.event_name, - hostname: Request.sanitize_hostname(host), - pathname: get_pathname(event.request.uri, event.request.hash_mode) + hostname: event.request.hostname, + pathname: event.request.pathname }) end @@ -239,7 +252,7 @@ defmodule Plausible.Ingestion.Event do %{event | clickhouse_event: valid_clickhouse_event} {:error, changeset} -> - drop(event, {:error, changeset}) + drop(event, :invalid, changeset: changeset) end end @@ -260,24 +273,10 @@ defmodule Plausible.Ingestion.Event do defp write_to_buffer(%__MODULE__{clickhouse_event: clickhouse_event} = event) do {:ok, _} = Plausible.Event.WriteBuffer.insert(clickhouse_event) + emit_telemetry_buffered() event end - defp get_pathname(_uri = nil, _hash_mode), do: "/" - - defp get_pathname(uri, hash_mode) do - pathname = - (uri.path || "/") - |> URI.decode() - |> String.trim_trailing() - - if hash_mode == 1 && uri.fragment do - pathname <> "#" <> URI.decode(uri.fragment) - else - pathname - end - end - defp parse_referrer(_uri, _referrer_str = nil), do: nil defp parse_referrer(uri, referrer_str) do @@ -409,8 +408,11 @@ defmodule Plausible.Ingestion.Event do defp spam_referrer?(_), do: false - defp domain_blocked?(domain) do - domain in Application.get_env(:plausible, :domain_blacklist) or - FunWithFlags.enabled?(:block_event_ingest, for: domain) + defp emit_telemetry_buffered() do + :telemetry.execute(telemetry_event_buffered(), %{}, %{}) + end + + defp emit_telemetry_dropped(reason) do + :telemetry.execute(telemetry_event_dropped(), %{}, %{reason: reason}) end end diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index 39838bf94..3264affdd 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -1,51 +1,68 @@ defmodule Plausible.Ingestion.Request do @moduledoc """ - The %Plausible.Ingestion.Request{} struct stores all needed fields to create an event downstream. + The %Plausible.Ingestion.Request{} struct stores all needed fields + to create an event downstream. Pre-eliminary validation is made + to detect user errors early. """ - defstruct [ - :remote_ip, - :user_agent, - :event_name, - :uri, - :referrer, - :domains, - :screen_width, - :hash_mode, - props: %{}, - query_params: %{} - ] - - @type t() :: %__MODULE__{ - remote_ip: String.t() | nil, - user_agent: String.t() | nil, - event_name: term(), - uri: URI.t() | nil, - referrer: term(), - domains: list(String.t()), - screen_width: term(), - hash_mode: term(), - props: map(), - query_params: map() - } - - @spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json} + use Ecto.Schema + alias Ecto.Changeset + + embedded_schema do + field :remote_ip, :string + field :user_agent, :string + field :event_name, :string + field :uri, :string + field :hostname, :string + field :referrer, :string + field :domains, {:array, :string} + field :screen_width, :string + field :hash_mode, :string + field :pathname, :string + field :props, :map + field :query_params, :map + + field :timestamp, :naive_datetime, + default: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) + end + + @type t() :: %__MODULE__{} + + @spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, Changeset.t()} @doc """ - Builds a list of %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. + Builds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. """ def build(%Plug.Conn{} = conn) do - with {:ok, request_body} <- parse_body(conn) do - %__MODULE__{} - |> Map.put(:remote_ip, PlausibleWeb.RemoteIp.get(conn)) - |> put_uri(request_body) - |> put_user_agent(conn) - |> put_request_params(request_body) - |> put_query_params() - |> map_domains(request_body) - |> then(&{:ok, &1}) + changeset = Changeset.change(%__MODULE__{}) + + case parse_body(conn) do + {:ok, request_body} -> + changeset + |> put_remote_ip(conn) + |> put_uri(request_body) + |> put_hostname() + |> put_user_agent(conn) + |> put_request_params(request_body) + |> put_pathname() + |> put_query_params() + |> map_domains(request_body) + |> Changeset.validate_required([ + :event_name, + :hostname, + :pathname, + :timestamp + ]) + |> Changeset.apply_action(nil) + + {:error, :invalid_json} -> + {:error, Changeset.add_error(changeset, :request, "Unable to parse request body as json")} end end + defp put_remote_ip(changeset, conn) do + Changeset.put_change(changeset, :remote_ip, PlausibleWeb.RemoteIp.get(conn)) + end + defp parse_body(conn) do case conn.body_params do %Plug.Conn.Unfetched{} -> @@ -61,38 +78,76 @@ defmodule Plausible.Ingestion.Request do end end - defp put_request_params(%__MODULE__{} = request, %{} = request_body) do - %__MODULE__{ - request - | event_name: request_body["n"] || request_body["name"], - referrer: request_body["r"] || request_body["referrer"], - screen_width: request_body["w"] || request_body["screen_width"], - hash_mode: request_body["h"] || request_body["hashMode"], - props: parse_props(request_body) - } - end - - defp map_domains(%__MODULE__{} = request, %{} = request_body) do - domains = - if raw = request_body["d"] || request_body["domain"] do - raw - |> String.split(",") - |> Enum.map(&sanitize_hostname/1) - else - [sanitize_hostname(request.uri)] - end + defp put_request_params(changeset, %{} = request_body) do + Changeset.change( + changeset, + event_name: request_body["n"] || request_body["name"], + referrer: request_body["r"] || request_body["referrer"], + screen_width: request_body["w"] || request_body["screen_width"], + hash_mode: request_body["h"] || request_body["hashMode"], + props: parse_props(request_body) + ) + end - %__MODULE__{request | domains: domains} + defp put_pathname(changeset) do + uri = Changeset.get_field(changeset, :uri) + hash_mode = Changeset.get_field(changeset, :hash_mode) + pathname = get_pathname(uri, hash_mode) + Changeset.put_change(changeset, :pathname, pathname) end - defp put_uri(%__MODULE__{} = request, %{} = request_body) do - if url = request_body["u"] || request_body["url"] do - %__MODULE__{request | uri: URI.parse(url)} - else - request + defp map_domains(changeset, %{} = request_body) do + raw = request_body["d"] || request_body["domain"] + raw = if is_binary(raw), do: String.trim(raw) + + case raw do + "" -> + Changeset.add_error(changeset, :domain, "can't be blank") + + raw when is_binary(raw) -> + domains = + raw + |> String.split(",") + |> Enum.map(&sanitize_hostname/1) + + Changeset.put_change(changeset, :domains, domains) + + nil -> + from_uri = sanitize_hostname(Changeset.get_field(changeset, :uri)) + + if from_uri do + Changeset.put_change(changeset, :domains, [from_uri]) + else + Changeset.add_error(changeset, :domain, "can't be blank") + end end end + defp put_uri(changeset, %{} = request_body) do + url = request_body["u"] || request_body["url"] + + case url do + nil -> + Changeset.add_error(changeset, :url, "is required") + + url when is_binary(url) -> + Changeset.put_change(changeset, :uri, URI.parse(url)) + + _ -> + Changeset.add_error(changeset, :url, "must be a string") + end + end + + defp put_hostname(changeset) do + host = + case Changeset.get_field(changeset, :uri) do + %{host: host} when is_binary(host) and host != "" -> host + _ -> "(none)" + end + + Changeset.put_change(changeset, :hostname, sanitize_hostname(host)) + end + defp parse_props(%{} = request_body) do raw_props = request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"] @@ -132,23 +187,38 @@ defmodule Plausible.Ingestion.Request do end end - defp put_query_params(%__MODULE__{} = request) do - case request do - %__MODULE__{uri: %URI{query: query}} when is_binary(query) -> - %__MODULE__{request | query_params: URI.decode_query(query)} + defp put_query_params(changeset) do + case Changeset.get_field(changeset, :uri) do + %{query: query} when is_binary(query) -> + Changeset.put_change(changeset, :query_params, URI.decode_query(query)) _any -> - request + changeset end end - defp put_user_agent(%__MODULE__{} = request, %Plug.Conn{} = conn) do + defp put_user_agent(changeset, %Plug.Conn{} = conn) do user_agent = conn |> Plug.Conn.get_req_header("user-agent") |> List.first() - %__MODULE__{request | user_agent: user_agent} + Changeset.put_change(changeset, :user_agent, user_agent) + end + + defp get_pathname(nil, _hash_mode), do: "/" + + defp get_pathname(uri, hash_mode) do + pathname = + (uri.path || "/") + |> URI.decode() + |> String.trim_trailing() + + if hash_mode == 1 && uri.fragment do + pathname <> "#" <> URI.decode(uri.fragment) + else + pathname + end end @doc """ diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex index 01ecb961d..42b5b484a 100644 --- a/lib/plausible/site/admin.ex +++ b/lib/plausible/site/admin.ex @@ -40,7 +40,16 @@ defmodule Plausible.SiteAdmin do timezone: nil, public: nil, owner: %{value: &get_owner_email/1}, - other_members: %{value: &get_other_members/1} + other_members: %{value: &get_other_members/1}, + limits: %{ + value: fn site -> + case site.ingest_rate_limit_threshold do + nil -> "" + 0 -> "🛑 BLOCKED" + n -> "⏱ #{n}/#{site.ingest_rate_limit_scale_seconds}s (per server)" + end + end + } ] end diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex index b77943764..8069d655f 100644 --- a/lib/plausible/site/cache.ex +++ b/lib/plausible/site/cache.ex @@ -19,23 +19,18 @@ defmodule Plausible.Site.Cache do database counterpart -- to spare bandwidth and query execution time, only selected database columns are retrieved and cached. - There are two modes of refreshing the cache: `:all` and `:single`. + There are two modes of refreshing the cache: `:all` and `:updated_recently`. * `:all` means querying the database for all Site entries and should be done - periodically (via `Cache.Warmer`). All existing Cache entries all cleared - prior to writing the new batch. + periodically (via `Cache.Warmer`). All stale Cache entries are cleared + upon persisting the new batch. - * `:single` attempts to re-query a specific site by domain and should be done - only when the initial Cache.get attempt resulted with `nil`. Single refresh will - write `%Ecto.NoResultsError{}` to the cache so that subsequent Cache.get calls - indicate that we already failed to retrieve a Site. + * `:updated_recently` attempts to re-query sites updated within the last + 15 minutes only, to account for most recent changes. This refresh + is lighter on the database and is meant to be executed more frequently + (via `Cache.Warmer.RecentlyUpdated`). - This helps in recognising missing/deleted Sites with minimal number of DB lookups - across a disconnected cluster within the periodic refresh window. - - Refreshing a single Site emits a telemetry event including `duration` measurement - and meta-data indicating whether the site was found in the DB or is missing still. - The telemetry event is defined with `telemetry_event_refresh/2`. + Refreshing the cache emits telemetry event defined as per `telemetry_event_refresh/2`. The `@cached_schema_fields` attribute defines the list of DB columns queried on each cache refresh. @@ -49,6 +44,7 @@ defmodule Plausible.Site.Cache do alias Plausible.Site @cache_name :sites_by_domain + @modes [:all, :updated_recently] @cached_schema_fields ~w( id @@ -57,8 +53,7 @@ defmodule Plausible.Site.Cache do ingest_rate_limit_threshold )a - @type not_found_in_db() :: %Ecto.NoResultsError{} - @type t() :: Site.t() | not_found_in_db() + @type t() :: Site.t() def name(), do: @cache_name @@ -75,22 +70,36 @@ defmodule Plausible.Site.Cache do @spec refresh_all(Keyword.t()) :: :ok def refresh_all(opts \\ []) do - cache_name = Keyword.get(opts, :cache_name, @cache_name) - - measure_duration(telemetry_event_refresh(cache_name, :all), fn -> - sites_by_domain_query = - from s in Site, - select: { - s.domain, - %{struct(s, ^@cached_schema_fields) | from_cache?: true} - } - - sites_by_domain = Plausible.Repo.all(sites_by_domain_query) - - :ok = merge(sites_by_domain, opts) - end) + sites_by_domain_query = + from s in Site, + select: { + s.domain, + %{struct(s, ^@cached_schema_fields) | from_cache?: true} + } + + refresh( + :all, + sites_by_domain_query, + Keyword.put(opts, :delete_stale_items?, true) + ) + end - :ok + @spec refresh_updated_recently(Keyword.t()) :: :ok + def refresh_updated_recently(opts \\ []) do + recently_updated_sites_query = + from s in Site, + order_by: [asc: s.updated_at], + where: s.updated_at > ago(^15, "minute"), + select: { + s.domain, + %{struct(s, ^@cached_schema_fields) | from_cache?: true} + } + + refresh( + :updated_recently, + recently_updated_sites_query, + Keyword.put(opts, :delete_stale_items?, false) + ) end @spec merge(new_items :: [Site.t()], opts :: Keyword.t()) :: :ok @@ -99,18 +108,20 @@ defmodule Plausible.Site.Cache do def merge(new_items, opts) do cache_name = Keyword.get(opts, :cache_name, @cache_name) - {:ok, old_keys} = Cachex.keys(cache_name) + true = Cachex.put_many!(cache_name, new_items) - new = MapSet.new(Enum.into(new_items, [], fn {k, _} -> k end)) - old = MapSet.new(old_keys) + if Keyword.get(opts, :delete_stale_items?, true) do + {:ok, old_keys} = Cachex.keys(cache_name) - true = Cachex.put_many!(cache_name, new_items) + new = MapSet.new(Enum.into(new_items, [], fn {k, _} -> k end)) + old = MapSet.new(old_keys) - old - |> MapSet.difference(new) - |> Enum.each(fn k -> - Cachex.del(cache_name, k) - end) + old + |> MapSet.difference(new) + |> Enum.each(fn k -> + Cachex.del(cache_name, k) + end) + end :ok end @@ -152,29 +163,8 @@ defmodule Plausible.Site.Cache do end end - @spec refresh_one(String.t(), Keyword.t()) :: {:ok, t()} | {:error, any()} - def refresh_one(domain, opts) do - cache_name = Keyword.get(opts, :cache_name, @cache_name) - force? = Keyword.get(opts, :force?, false) - - if not enabled?() and not force?, do: raise("Cache: '#{cache_name}' is disabled") - - measure_duration_with_metadata(telemetry_event_refresh(cache_name, :one), fn -> - {found_in_db?, item_to_cache} = select_one(domain) - - case Cachex.put(cache_name, domain, item_to_cache) do - {:ok, _} -> - result = {:ok, item_to_cache} - {result, with_telemetry_metadata(found_in_db?: found_in_db?)} - - {:error, _} = error -> - {error, with_telemetry_metadata(error: true)} - end - end) - end - - @spec telemetry_event_refresh(atom(), :all | :one) :: list(atom()) - def telemetry_event_refresh(cache_name, mode) when mode in [:all, :one] do + @spec telemetry_event_refresh(atom(), atom()) :: list(atom()) + def telemetry_event_refresh(cache_name \\ @cache_name, mode) when mode in @modes do [:plausible, :cache, cache_name, :refresh, mode] end @@ -182,26 +172,15 @@ defmodule Plausible.Site.Cache do Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true end - defp select_one(domain) do - site_by_domain_query = - from s in Site, - where: s.domain == ^domain, - select: %{struct(s, ^@cached_schema_fields) | from_cache?: true} - - case Plausible.Repo.one(site_by_domain_query) do - nil -> {false, %Ecto.NoResultsError{}} - site -> {true, site} - end - end + defp refresh(mode, query, opts) when mode in @modes do + cache_name = Keyword.get(opts, :cache_name, @cache_name) - defp with_telemetry_metadata(props) do - Enum.into(props, %{}) - end + measure_duration(telemetry_event_refresh(cache_name, mode), fn -> + sites_by_domain = Plausible.Repo.all(query) + :ok = merge(sites_by_domain, opts) + end) - defp measure_duration_with_metadata(event, fun) when is_function(fun, 0) do - {duration, {result, telemetry_metadata}} = time_it(fun) - :telemetry.execute(event, %{duration: duration}, telemetry_metadata) - result + :ok end defp measure_duration(event, fun) when is_function(fun, 0) do diff --git a/lib/plausible/site/cache/rate_limiter.ex b/lib/plausible/site/cache/rate_limiter.ex deleted file mode 100644 index dc11feb5e..000000000 --- a/lib/plausible/site/cache/rate_limiter.ex +++ /dev/null @@ -1,109 +0,0 @@ -defmodule Plausible.Site.RateLimiter do - @policy_for_non_existing_sites :deny - @policy_on_rate_limiting_backend_error :allow - - @moduledoc """ - Thin wrapper around Hammer for rate limiting domain-specific events - during the ingestion phase. Currently there are two policies - on which the `allow/2` function operates: - * `:allow` - * `:deny` - - Rate Limiting buckets are configured per site (externally via the CRM). - See: `Plausible.Site` - - To look up each site's configuration, the RateLimiter fetches - a Site by domain using `Plausible.Cache` interface. - If the Site is not found in Cache, a DB refresh attempt is made. - The result of that last attempt gets stored in Cache to prevent - excessive DB queries. - - The module defines two policies outside the regular bucket inspection: - * when the site does not exist in the database: #{@policy_for_non_existing_sites} - * when the underlying rate limiting mechanism returns - an internal error: #{@policy_on_rate_limiting_backend_error} - - Each policy computation emits a single telemetry event. - See: `policy_telemetry_event/1` - """ - alias Plausible.Site - alias Plausible.Site.Cache - - require Logger - - @spec allow?(String.t(), Keyword.t()) :: boolean() - def allow?(domain, opts \\ []) do - policy(domain, opts) == :allow - end - - @spec key(String.t()) :: String.t() - def key(domain) do - "ingest:site:#{domain}" - end - - @spec policy_telemetry_event(:allow | :deny) :: list(atom()) - def policy_telemetry_event(policy) do - [:plausible, :ingest, :rate_limit, policy] - end - - defp policy(domain, opts) do - result = - case get_from_cache_or_refresh(domain, Keyword.get(opts, :cache_opts, [])) do - %Ecto.NoResultsError{} -> - @policy_for_non_existing_sites - - %Site{} = site -> - check_rate_limit(site, opts) - - {:error, _} -> - @policy_on_rate_limiting_backend_error - end - - :ok = emit_allowance_telemetry(result) - result - end - - defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do - :allow - end - - defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts) - when is_integer(threshold) do - key = Keyword.get(opts, :key, key(site.domain)) - scale_ms = site.ingest_rate_limit_scale_seconds * 1_000 - - case Hammer.check_rate(key, scale_ms, threshold) do - {:deny, _} -> - :deny - - {:allow, _} -> - :allow - - {:error, reason} -> - Logger.error( - "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}" - ) - - @policy_on_rate_limiting_backend_error - end - end - - defp get_from_cache_or_refresh(domain, cache_opts) do - case Cache.get(domain, cache_opts) do - %Site{} = site -> - site - - %Ecto.NoResultsError{} = not_found -> - not_found - - nil -> - with {:ok, refreshed_item} <- Cache.refresh_one(domain, cache_opts) do - refreshed_item - end - end - end - - defp emit_allowance_telemetry(policy) do - :telemetry.execute(policy_telemetry_event(policy), %{}, %{}) - end -end diff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.ex index cd31863eb..4ef4489c0 100644 --- a/lib/plausible/site/cache/warmer.ex +++ b/lib/plausible/site/cache/warmer.ex @@ -30,11 +30,11 @@ defmodule Plausible.Site.Cache.Warmer do @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore def child_spec(opts) do - child_name = Keyword.get(opts, :child_name, {:local, __MODULE__}) + child_name = Keyword.get(opts, :child_name, __MODULE__) %{ - id: __MODULE__, - start: {:gen_cycle, :start_link, [child_name, __MODULE__, opts]} + id: child_name, + start: {:gen_cycle, :start_link, [{:local, child_name}, __MODULE__, opts]} } end diff --git a/lib/plausible/site/cache/warmer/all.ex b/lib/plausible/site/cache/warmer/all.ex new file mode 100644 index 000000000..021c099ad --- /dev/null +++ b/lib/plausible/site/cache/warmer/all.ex @@ -0,0 +1,8 @@ +defmodule Plausible.Site.Cache.Warmer.All do + @moduledoc """ + A Cache.Warmer adapter that refreshes the Sites Cache fully. + This module exists only to make it explicit what the warmer + refreshes, to be used in the supervisor tree. + """ + defdelegate child_spec(opts), to: Plausible.Site.Cache.Warmer +end diff --git a/lib/plausible/site/cache/warmer/recently_updated.ex b/lib/plausible/site/cache/warmer/recently_updated.ex new file mode 100644 index 000000000..2f3e37806 --- /dev/null +++ b/lib/plausible/site/cache/warmer/recently_updated.ex @@ -0,0 +1,20 @@ +defmodule Plausible.Site.Cache.Warmer.RecentlyUpdated do + @moduledoc """ + A Cache.Warmer adapter that only refreshes the Cache + with recently updated sites every 30 seconds. + """ + alias Plausible.Site.Cache + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore + def child_spec(opts) do + child_name = Keyword.get(opts, :child_name, __MODULE__) + + opts = [ + child_name: child_name, + interval: :timer.seconds(30), + warmer_fn: &Cache.refresh_updated_recently/1 + ] + + Plausible.Site.Cache.Warmer.child_spec(opts) + end +end diff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.ex new file mode 100644 index 000000000..d0e4fe6ef --- /dev/null +++ b/lib/plausible/site/gate_keeper.ex @@ -0,0 +1,87 @@ +defmodule Plausible.Site.GateKeeper do + @type policy() :: :allow | :not_found | :block | :throttle + @policy_for_non_existing_sites :not_found + @policy_on_rate_limiting_backend_error :allow + + @type t() :: :allow | {:deny, policy()} + + @moduledoc """ + Thin wrapper around Hammer for gate keeping domain-specific events + during the ingestion phase. When the site is allowed, gate keeping + check returns `:allow`, otherwise a `:deny` tagged tuple is returned + with one of the following policy markers: + * `:not_found` (indicates site not found in cache) + * `:block` (indicates disabled sites) + * `:throttle` (indicates rate limiting) + + Rate Limiting buckets are configured per site (externally via the CRM). + See: `Plausible.Site` + + To look up each site's configuration, the RateLimiter fetches + a Site by domain using `Plausible.Cache` interface. + + The module defines two policies outside the regular bucket inspection: + * when the the site is not found in cache: #{@policy_for_non_existing_sites} + * when the underlying rate limiting mechanism returns + an internal error: #{@policy_on_rate_limiting_backend_error} + """ + alias Plausible.Site + alias Plausible.Site.Cache + + require Logger + + @spec check(String.t(), Keyword.t()) :: t() + def check(domain, opts \\ []) when is_binary(domain) do + case policy(domain, opts) do + :allow -> :allow + other -> {:deny, other} + end + end + + @spec key(String.t()) :: String.t() + def key(domain) do + "ingest:site:#{domain}" + end + + defp policy(domain, opts) when is_binary(domain) do + result = + case Cache.get(domain, Keyword.get(opts, :cache_opts, [])) do + nil -> + @policy_for_non_existing_sites + + %Site{} = site -> + check_rate_limit(site, opts) + end + + result + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil}, _opts) do + :allow + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do + :block + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts) + when is_integer(threshold) do + key = Keyword.get(opts, :key, key(site.domain)) + scale_ms = site.ingest_rate_limit_scale_seconds * 1_000 + + case Hammer.check_rate(key, scale_ms, threshold) do + {:deny, _} -> + :throttle + + {:allow, _} -> + :allow + + {:error, reason} -> + Logger.error( + "Error checking rate limit for '#{key}': #{inspect(reason)}. Falling back to: #{@policy_on_rate_limiting_backend_error}" + ) + + @policy_on_rate_limiting_backend_error + end + end +end diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex index fa75a9f57..c5cb11265 100644 --- a/lib/plausible/telemetry/plausible_metrics.ex +++ b/lib/plausible/telemetry/plausible_metrics.ex @@ -3,6 +3,8 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do Custom PromEx plugin for instrumenting code within Plausible app. """ use PromEx.Plugin + alias Plausible.Site + alias Plausible.Ingestion @impl true def polling_metrics(opts) do @@ -19,6 +21,45 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do ] end + @impl true + def event_metrics(opts) do + otp_app = Keyword.fetch!(opts, :otp_app) + metric_prefix = Keyword.get(opts, :metric_prefix, PromEx.metric_prefix(otp_app, :plausible)) + + Event.build( + :plausible_internal_telemetry, + [ + distribution( + metric_prefix ++ [:cache_warmer, :sites, :refresh, :all], + event_name: Site.Cache.telemetry_event_refresh(:all), + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:cache_warmer, :sites, :refresh, :updated_recently], + event_name: Site.Cache.telemetry_event_refresh(:updated_recently), + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + counter( + metric_prefix ++ [:ingest, :events, :buffered, :total], + event_name: Ingestion.Event.telemetry_event_buffered() + ), + counter( + metric_prefix ++ [:ingest, :events, :dropped, :total], + event_name: Ingestion.Event.telemetry_event_dropped(), + tags: [:reason] + ) + ] + ) + end + @doc """ Add telemetry events for Session and Event write buffers """ @@ -68,6 +109,11 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do count: sessions_count, hit_rate: sessions_hit_rate }) + + :telemetry.execute([:prom_ex, :plugin, :cachex, :sites], %{ + count: Site.Cache.size(), + hit_rate: Site.Cache.hit_rate() + }) end defp write_buffer_metrics(metric_prefix, poll_rate) do @@ -109,13 +155,21 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do last_value( metric_prefix ++ [:cache, :user_agents, :hit_ratio], event_name: [:prom_ex, :plugin, :cachex, :user_agents], - description: "UA cache hit ratio", measurement: :hit_rate ), last_value( metric_prefix ++ [:cache, :sessions, :hit_ratio], event_name: [:prom_ex, :plugin, :cachex, :sessions], - description: "Sessions cache hit ratio", + measurement: :hit_rate + ), + last_value( + metric_prefix ++ [:cache, :sites, :size], + event_name: [:prom_ex, :plugin, :cachex, :sites], + measurement: :count + ), + last_value( + metric_prefix ++ [:cache, :sites, :hit_ratio], + event_name: [:prom_ex, :plugin, :cachex, :sites], measurement: :hit_rate ) ] diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index 82d1281e6..f81b4944a 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -35,10 +35,10 @@ defmodule PlausibleWeb.Api.ExternalController do end end else - {:error, :invalid_json} -> + {:error, %Ecto.Changeset{} = changeset} -> conn |> put_status(400) - |> json(%{errors: %{request: "Unable to parse request body as json"}}) + |> json(%{errors: traverse_errors(changeset)}) end end diff --git a/test/plausible/ingestion/event_telemetry_test.exs b/test/plausible/ingestion/event_telemetry_test.exs new file mode 100644 index 000000000..8c50698b4 --- /dev/null +++ b/test/plausible/ingestion/event_telemetry_test.exs @@ -0,0 +1,44 @@ +defmodule Plausible.Ingestion.EventTelemetryTest do + import Phoenix.ConnTest + + alias Plausible.Ingestion.Request + alias Plausible.Ingestion.Event + + use Plausible.DataCase, async: false + + test "telemetry is emitted for all events", %{test: test} do + test_pid = self() + + telemetry_dropped = Event.telemetry_event_dropped() + telemetry_buffered = Event.telemetry_event_buffered() + + :telemetry.attach_many( + "#{test}-telemetry-handler", + [ + telemetry_dropped, + telemetry_buffered + ], + fn event, %{}, metadata, _ -> + send(test_pid, {:telemetry_handled, event, metadata}) + end, + %{} + ) + + site = insert(:site, ingest_rate_limit_threshold: 2) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + d: "#{site.domain}" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + for _ <- 1..3, do: Event.build_and_buffer(request) + + assert_receive {:telemetry_handled, ^telemetry_buffered, %{}} + assert_receive {:telemetry_handled, ^telemetry_buffered, %{}} + assert_receive {:telemetry_handled, ^telemetry_dropped, %{reason: :throttle}} + end +end diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 5679efadc..c2e997c8c 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -1,112 +1,102 @@ defmodule Plausible.Ingestion.EventTest do - use Plausible.DataCase - - @valid_request %Plausible.Ingestion.Request{ - remote_ip: "2.2.2.2", - user_agent: - "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405", - event_name: "pageview", - uri: URI.parse("http://skywalker.test"), - referrer: "http://m.facebook.test/", - screen_width: 1440, - hash_mode: nil, - query_params: %{ - "utm_medium" => "utm_medium", - "utm_source" => "utm_source", - "utm_campaign" => "utm_campaign", - "utm_content" => "utm_content", - "utm_term" => "utm_term", - "source" => "source", - "ref" => "ref" + use Plausible.DataCase, async: true + + import Phoenix.ConnTest + + alias Plausible.Ingestion.Request + alias Plausible.Ingestion.Event + + test "event pipeline processes a request into an event" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://#{site.domain}" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request) + end + + test "event pipeline drops a request when site does not exists" do + payload = %{ + name: "pageview", + url: "http://dummy.site" } - } - - describe "integration" do - test "build_and_buffer/1 creates an event" do - assert {:ok, %{buffered: [_], dropped: []}} = - @valid_request - |> Map.put(:domains, ["plausible-ingestion-event-basic.test"]) - |> Plausible.Ingestion.Event.build_and_buffer() - - assert %Plausible.ClickhouseEvent{ - session_id: session_id, - user_id: user_id, - domain: "plausible-ingestion-event-basic.test", - browser: "Safari", - browser_version: "", - city_geoname_id: 2_988_507, - country_code: "FR", - hostname: "skywalker.test", - "meta.key": [], - "meta.value": [], - name: "pageview", - operating_system: "iOS", - operating_system_version: "3.2", - pathname: "/", - referrer: "m.facebook.test", - referrer_source: "utm_source", - screen_size: "Desktop", - subdivision1_code: "FR-IDF", - subdivision2_code: "FR-75", - transferred_from: "", - utm_campaign: "utm_campaign", - utm_content: "utm_content", - utm_medium: "utm_medium", - utm_source: "utm_source", - utm_term: "utm_term" - } = get_event("plausible-ingestion-event-basic.test") - - assert is_integer(session_id) - assert is_integer(user_id) - end - - test "build_and_buffer/1 takes multiple domains" do - request = %Plausible.Ingestion.Request{ - @valid_request - | domains: [ - "plausible-ingestion-event-multiple-1.test", - "plausible-ingestion-event-multiple-2.test" - ] - } - - assert {:ok, %{buffered: [_, _], dropped: []}} = - Plausible.Ingestion.Event.build_and_buffer(request) - - assert %Plausible.ClickhouseEvent{domain: "plausible-ingestion-event-multiple-1.test"} = - get_event("plausible-ingestion-event-multiple-1.test") - - assert %Plausible.ClickhouseEvent{domain: "plausible-ingestion-event-multiple-2.test"} = - get_event("plausible-ingestion-event-multiple-2.test") - end - - test "build_and_buffer/1 drops invalid events" do - request = %Plausible.Ingestion.Request{ - @valid_request - | domains: ["plausible-ingestion-event-multiple-with-error-1.test", nil] - } - - assert {:ok, %{buffered: [_], dropped: [dropped]}} = - Plausible.Ingestion.Event.build_and_buffer(request) - - assert {:error, changeset} = dropped.drop_reason - refute changeset.valid? - - assert %Plausible.ClickhouseEvent{ - domain: "plausible-ingestion-event-multiple-with-error-1.test" - } = get_event("plausible-ingestion-event-multiple-with-error-1.test") - end - - defp get_event(domain) do - Plausible.TestUtils.eventually(fn -> - Plausible.Event.WriteBuffer.flush() - - event = - Plausible.ClickhouseRepo.one( - from e in Plausible.ClickhouseEvent, where: e.domain == ^domain - ) - - {!is_nil(event), event} - end) - end + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :not_found + end + + test "event pipeline drops a request when referrer is spam" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + referrer: "https://www.1-best-seo.com", + domain: site.domain + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :spam_referrer + end + + test "event pipeline drops a request when referrer is spam for multiple domains" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + referrer: "https://www.1-best-seo.com", + d: "#{site.domain},#{site.domain}" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{dropped: [dropped, dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :spam_referrer + end + + test "event pipeline selectively drops an event for multiple domains" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + d: "#{site.domain},thisdoesnotexist.com" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [_], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :not_found + end + + test "event pipeline selectively drops an event when rate-limited" do + site = insert(:site, ingest_rate_limit_threshold: 1) + + payload = %{ + name: "pageview", + url: "http://dummy.site", + d: "#{site.domain}" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [_], dropped: []}} = Event.build_and_buffer(request) + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :throttle end end diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs new file mode 100644 index 000000000..10d529ac2 --- /dev/null +++ b/test/plausible/ingestion/request_test.exs @@ -0,0 +1,187 @@ +defmodule Plausible.Ingestion.RequestTest do + use Plausible.DataCase, async: true + + import Phoenix.ConnTest + import Plug.Conn + + alias Plausible.Ingestion.Request + + test "request cannot be built from conn without payload" do + conn = build_conn(:post, "/api/events", %{}) + assert {:error, changeset} = Request.build(conn) + + errors = Keyword.keys(changeset.errors) + assert :event_name in errors + assert :domain in errors + assert :url in errors + end + + test "request cannot be built from non-json payload" do + conn = build_conn(:post, "/api/events", "defnotjson") + assert {:error, changeset} = Request.build(conn) + assert changeset.errors[:request] + end + + test "request can be built from URL alone" do + payload = %{ + name: "pageview", + url: "http://dummy.site" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + + assert request.domains == ["dummy.site"] + assert request.event_name == "pageview" + assert request.pathname == "/" + assert request.remote_ip == "127.0.0.1" + assert %NaiveDateTime{} = request.timestamp + assert request.user_agent == nil + assert request.hostname == "dummy.site" + assert request.uri.host == "dummy.site" + assert request.uri.scheme == "http" + assert request.props == %{} + end + + test "request can be built with domain" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + assert request.domains == ["dummy.site"] + assert request.uri.host == "dummy.site" + end + + test "request can be built with domain using shorthands" do + payload = %{ + n: "pageview", + d: "dummy.site", + u: "http://dummy.site/index" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + assert request.domains == ["dummy.site"] + assert request.uri.host == "dummy.site" + end + + test "request can be built for multiple domains" do + payload = %{ + n: "pageview", + d: "dummy.site,crash.site", + u: "http://dummy.site/index" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + assert request.domains == ["dummy.site", "crash.site"] + assert request.uri.host == "dummy.site" + end + + test "hostname is (none) if host-less uri provided" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "about:config" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + assert request.hostname == "(none)" + end + + test "hostname is set" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html" + } + + conn = build_conn(:post, "/api/events", payload) + assert {:ok, request} = Request.build(conn) + assert request.hostname == "dummy.site" + end + + test "user agent is set as-is" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html" + } + + conn = build_conn(:post, "/api/events", payload) |> put_req_header("user-agent", "Mozilla") + assert {:ok, request} = Request.build(conn) + assert request.user_agent == "Mozilla" + end + + test "request params are set" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html", + referrer: "https://example.com", + screen_width: 1024, + hashMode: 1, + props: %{ + "custom1" => "property1", + "custom2" => "property2" + } + } + + conn = build_conn(:post, "/api/events", payload) + + assert {:ok, request} = Request.build(conn) + assert request.referrer == "https://example.com" + assert request.screen_width == 1024 + assert request.hash_mode == 1 + assert request.props["custom1"] == "property1" + assert request.props["custom2"] == "property2" + end + + test "pathname is set" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/pictures/index.html#foo" + } + + conn = build_conn(:post, "/api/events", payload) + + assert {:ok, request} = Request.build(conn) + + assert request.pathname == "/pictures/index.html" + end + + test "pathname is set with hashMode" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/pictures/index.html#foo", + hashMode: 1 + } + + conn = build_conn(:post, "/api/events", payload) + + assert {:ok, request} = Request.build(conn) + + assert request.pathname == "/pictures/index.html#foo" + end + + test "query params are set" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/pictures/index.html?foo=bar&baz=bam" + } + + conn = build_conn(:post, "/api/events", payload) + + assert {:ok, request} = Request.build(conn) + assert request.query_params["foo"] == "bar" + assert request.query_params["baz"] == "bam" + end +end diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs index 84648c66a..a6a34ca13 100644 --- a/test/plausible/site/cache_test.exs +++ b/test/plausible/site/cache_test.exs @@ -66,7 +66,7 @@ defmodule Plausible.Site.CacheTest do assert Cache.hit_rate(test) == 50 end - test "a single cached site can be refreshed", %{test: test} do + test "only recently updated sites can be refreshed", %{test: test} do {:ok, _} = start_test_cache(test) domain1 = "site1.example.com" @@ -74,44 +74,18 @@ defmodule Plausible.Site.CacheTest do cache_opts = [cache_name: test, force?: true] - assert Cache.get(domain1) == nil + yesterday = DateTime.utc_now() |> DateTime.add(-1 * 60 * 60 * 24) + insert(:site, domain: domain1, inserted_at: yesterday, updated_at: yesterday) - insert(:site, domain: domain1) + insert(:site, domain: domain2) - assert {:ok, %{domain: ^domain1}} = Cache.refresh_one(domain1, cache_opts) - assert %Site{domain: ^domain1} = Cache.get(domain1, cache_opts) + assert Cache.get(domain1, cache_opts) == nil + assert Cache.get(domain2, cache_opts) == nil - assert {:ok, %Ecto.NoResultsError{}} = Cache.refresh_one(domain2, cache_opts) - assert %Ecto.NoResultsError{} = Cache.get(domain2, cache_opts) - end - - test "refreshing a single site sends a telemetry event indicating record not found in the database", - %{ - test: test - } do - :ok = - start_test_cache_with_telemetry_handler(test, - event: Cache.telemetry_event_refresh(test, :one) - ) - - Cache.refresh_one("missing.example.com", force?: true, cache_name: test) - assert_receive {:telemetry_handled, %{found_in_db?: false}} - end - - test "refreshing a single site sends a telemetry event indicating record found in the database", - %{ - test: test - } do - domain = "site1.example.com" - insert(:site, domain: domain) - - :ok = - start_test_cache_with_telemetry_handler(test, - event: Cache.telemetry_event_refresh(test, :one) - ) + assert :ok = Cache.refresh_updated_recently(cache_opts) - Cache.refresh_one(domain, force?: true, cache_name: test) - assert_receive {:telemetry_handled, %{found_in_db?: true}} + refute Cache.get(domain1, cache_opts) + assert %Site{domain: ^domain2} = Cache.get(domain2, cache_opts) end test "refreshing all sites sends a telemetry event", @@ -190,8 +164,8 @@ defmodule Plausible.Site.CacheTest do assert Cache.get(domain2, cache_opts) refute Cache.get(domain1, cache_opts) - Cache.refresh_one(domain1, cache_opts) - assert Cache.get(domain1, cache_opts) == %Ecto.NoResultsError{} + :ok = Cache.refresh_all(cache_opts) + refute Cache.get(domain1, cache_opts) end end @@ -221,6 +195,16 @@ defmodule Plausible.Site.CacheTest do assert Cache.get("item2", cache_name: test, force?: true) end + test "merging optionally leaves stale items intact", %{test: test} do + {:ok, _} = start_test_cache(test) + + :ok = Cache.merge([{"item1", :item1}], cache_name: test) + :ok = Cache.merge([{"item2", :item2}], cache_name: test, delete_stale_items?: false) + + assert Cache.get("item1", cache_name: test, force?: true) + assert Cache.get("item2", cache_name: test, force?: true) + end + test "merging updates changed items", %{test: test} do {:ok, _} = start_test_cache(test) @@ -271,7 +255,7 @@ defmodule Plausible.Site.CacheTest do end defp start_test_warmer(opts) do - child_name_opt = {:child_name, {:local, Keyword.fetch!(opts, :cache_name)}} + child_name_opt = {:child_name, Keyword.fetch!(opts, :cache_name)} %{start: {m, f, a}} = Cache.Warmer.child_spec([child_name_opt | opts]) apply(m, f, a) end diff --git a/test/plausible/site/rate_limiter_test.exs b/test/plausible/site/gate_keeper_test.exs similarity index 60% rename from test/plausible/site/rate_limiter_test.exs rename to test/plausible/site/gate_keeper_test.exs index 74b8d92df..e67b229bf 100644 --- a/test/plausible/site/rate_limiter_test.exs +++ b/test/plausible/site/gate_keeper_test.exs @@ -1,8 +1,8 @@ -defmodule Plausible.Site.RateLimiterTest do +defmodule Plausible.Site.GateKeeperTest do use Plausible.DataCase, async: true alias Plausible.Site.Cache - alias Plausible.Site.RateLimiter + alias Plausible.Site.GateKeeper import ExUnit.CaptureLog @@ -12,27 +12,15 @@ defmodule Plausible.Site.RateLimiterTest do {:ok, %{opts: opts}} end - test "(for now) throws an exception when cache is disabled" do - assert_raise(RuntimeError, fn -> RateLimiter.allow?("example.com") end) - end - - test "sites not found in cache/DB are denied", %{opts: opts} do - refute RateLimiter.allow?("example.com", opts) + test "sites not found in cache are denied", %{opts: opts} do + assert {:deny, :not_found} = GateKeeper.check("example.com", opts) end test "site from cache with no ingest_rate_limit_threshold is allowed", %{test: test, opts: opts} do domain = "site1.example.com" add_site_and_refresh_cache(test, domain: domain) - assert RateLimiter.allow?(domain, opts) - end - - test "site from DB with no ingest_rate_limit_threshold is allowed", %{opts: opts} do - domain = "site1.example.com" - - insert(:site, domain: domain) - - assert RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) end test "rate limiting works with threshold", %{test: test, opts: opts} do @@ -44,9 +32,9 @@ defmodule Plausible.Site.RateLimiterTest do ingest_rate_limit_scale_seconds: 60 ) - assert RateLimiter.allow?(domain, opts) - refute RateLimiter.allow?(domain, opts) - refute RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) + assert {:deny, :throttle} = GateKeeper.check(domain, opts) + assert {:deny, :throttle} = GateKeeper.check(domain, opts) end test "rate limiting works with scale window", %{test: test, opts: opts} do @@ -58,11 +46,11 @@ defmodule Plausible.Site.RateLimiterTest do ingest_rate_limit_scale_seconds: 1 ) - assert RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) Process.sleep(1) - refute RateLimiter.allow?(domain, opts) + assert {:deny, :throttle} = GateKeeper.check(domain, opts) Process.sleep(1_000) - assert RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) end test "rate limiting prioritises cache lookups", %{test: test, opts: opts} do @@ -80,9 +68,9 @@ defmodule Plausible.Site.RateLimiterTest do # is completely empty insert(:site) - assert RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) :ok = Cache.refresh_all(opts[:cache_opts]) - refute RateLimiter.allow?(domain, opts) + assert {:deny, :not_found} = GateKeeper.check(domain, opts) end test "rate limiter policy switches to allow when RL backend errors bubble-up", %{ @@ -94,38 +82,24 @@ defmodule Plausible.Site.RateLimiterTest do site = add_site_and_refresh_cache(test, domain: domain, - ingest_rate_limit_threshold: 0, + ingest_rate_limit_threshold: 1, ingest_rate_limit_scale_seconds: 600 ) - refute RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) + assert {:deny, :throttle} = GateKeeper.check(domain, opts) + {:ok, :broken} = break_hammer(site) log = capture_log(fn -> - assert RateLimiter.allow?(domain, opts) + assert :allow = GateKeeper.check(domain, opts) end) assert log =~ "Error checking rate limit for 'ingest:site:causingerrors.example.com'" assert log =~ "Falling back to: allow" end - test "telemetry event is emitted on :deny", %{test: test, opts: opts} do - start_telemetry_handler(test, event: RateLimiter.policy_telemetry_event(:deny)) - RateLimiter.allow?("example.com", opts) - assert_receive :telemetry_handled - end - - test "telemetry event is emitted on :allow", %{test: test, opts: opts} do - start_telemetry_handler(test, event: RateLimiter.policy_telemetry_event(:allow)) - - domain = "site1.example.com" - add_site_and_refresh_cache(test, domain: domain) - - RateLimiter.allow?(domain, opts) - assert_receive :telemetry_handled - end - # We need a way to force Hammer to error-out on Hammer.check_rate/3. # This is tricky because we don't configure multiple backends, # so the easiest (and naive) way to do it, without mocking, is to @@ -137,7 +111,7 @@ defmodule Plausible.Site.RateLimiterTest do defp break_hammer(site) do scale_ms = site.ingest_rate_limit_scale_seconds * 1_000 rogue_key = site.domain - our_key = RateLimiter.key(rogue_key) + our_key = GateKeeper.key(rogue_key) {_, key} = Hammer.Utils.stamp_key(our_key, scale_ms) true = :ets.insert(:hammer_ets_buckets, {key, 1, "TOTALLY-WRONG", "ABSOLUTELY-BREAKING"}) {:ok, :broken} @@ -150,20 +124,7 @@ defmodule Plausible.Site.RateLimiterTest do defp add_site_and_refresh_cache(cache_name, site_data) do site = insert(:site, site_data) - Cache.refresh_one(site.domain, cache_name: cache_name, force?: true) + Cache.refresh_updated_recently(cache_name: cache_name, force?: true) site end - - defp start_telemetry_handler(test, event: event) do - test_pid = self() - - :telemetry.attach( - "#{test}-telemetry-handler", - event, - fn ^event, %{}, %{}, _ -> - send(test_pid, :telemetry_handled) - end, - %{} - ) - end end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index a9f427fa5..eae881be3 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -2,32 +2,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do use PlausibleWeb.ConnCase use Plausible.ClickhouseRepo - defp get_event(domain) do - Plausible.Event.WriteBuffer.flush() - - ClickhouseRepo.one( - from e in Plausible.ClickhouseEvent, - where: e.domain == ^domain, - order_by: [desc: e.timestamp] - ) - end - - defp get_events(domain) do - Plausible.Event.WriteBuffer.flush() - - ClickhouseRepo.all( - from e in Plausible.ClickhouseEvent, - where: e.domain == ^domain, - order_by: [desc: e.timestamp] - ) - end - @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" describe "POST /api/event" do - test "records the event", %{conn: conn} do + setup do + site1 = insert(:site) + {:ok, domain: site1.domain} + end + + test "records the event", %{conn: conn, domain: domain} do params = %{ - domain: "external-controller-test-1.com", + domain: domain, name: "pageview", url: "http://gigride.live/", referrer: "http://m.facebook.com/", @@ -39,17 +24,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-1.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.hostname == "gigride.live" - assert pageview.domain == "external-controller-test-1.com" + assert pageview.domain == domain assert pageview.pathname == "/" end - test "works with Content-Type: text/plain", %{conn: conn} do + test "works with Content-Type: text/plain", %{conn: conn, domain: domain} do params = %{ - domain: "external-controller-test-text-plain.com", + domain: domain, name: "pageview", url: "http://gigride.live/", screen_width: 1440 @@ -60,11 +45,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("content-type", "text/plain") |> post("/api/event", Jason.encode!(params)) - pageview = get_event("external-controller-test-text-plain.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.hostname == "gigride.live" - assert pageview.domain == "external-controller-test-text-plain.com" + assert pageview.domain == domain assert pageview.pathname == "/" end @@ -77,12 +62,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 400 end - test "can send to multiple dashboards by listing multiple domains", %{conn: conn} do + test "can send to multiple dashboards by listing multiple domains", %{ + conn: conn, + domain: domain1 + } do + domain2 = insert(:site).domain + params = %{ name: "pageview", url: "http://gigride.live/", referrer: "http://m.facebook.com/", - domain: "test-domain1.com,test-domain2.com", + domain: "#{domain1},#{domain2}", screen_width: 1440 } @@ -92,75 +82,75 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) assert response(conn, 202) == "ok" - assert get_event("test-domain1.com") - assert get_event("test-domain2.com") + assert get_event(domain1) + assert get_event(domain2) end - test "www. is stripped from domain", %{conn: conn} do + test "www. is stripped from domain", %{conn: conn, domain: domain} do params = %{ name: "custom event", url: "http://gigride.live/", - domain: "www.external-controller-test-2.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-2.com") + pageview = get_event(domain) - assert pageview.domain == "external-controller-test-2.com" + assert pageview.domain == domain end - test "www. is stripped from hostname", %{conn: conn} do + test "www. is stripped from hostname", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/", - domain: "external-controller-test-3.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-3.com") + pageview = get_event(domain) assert pageview.hostname == "example.com" end - test "empty path defaults to /", %{conn: conn} do + test "empty path defaults to /", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com", - domain: "external-controller-test-4.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-4.com") + pageview = get_event(domain) assert pageview.pathname == "/" end - test "trailing whitespace is removed", %{conn: conn} do + test "trailing whitespace is removed", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/path ", - domain: "external-controller-test-trailing-whitespace.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-trailing-whitespace.com") + pageview = get_event(domain) assert pageview.pathname == "/path" end - test "bots and crawlers are ignored", %{conn: conn} do + test "bots and crawlers are ignored", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/", - domain: "external-controller-test-5.com" + domain: domain } conn @@ -170,11 +160,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert get_event("external-controller-test-5.com") == nil end - test "Headless Chrome is ignored", %{conn: conn} do + test "Headless Chrome is ignored", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/", - domain: "headless-chrome-test.com" + domain: domain } conn @@ -184,14 +174,14 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do ) |> post("/api/event", params) - assert get_event("headless-chrome-test.com") == nil + assert get_event(domain) == nil end - test "parses user_agent", %{conn: conn} do + test "parses user_agent", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", - domain: "external-controller-test-6.com" + domain: domain } conn = @@ -199,7 +189,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-6.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.operating_system == "Mac" @@ -208,12 +198,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.browser_version == "70.0" end - test "parses referrer", %{conn: conn} do + test "parses referrer", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https://facebook.com", - domain: "external-controller-test-7.com" + domain: domain } conn = @@ -221,18 +211,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-7.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "Facebook" end - test "strips trailing slash from referrer", %{conn: conn} do + test "strips trailing slash from referrer", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https://facebook.com/page/", - domain: "external-controller-test-8.com" + domain: domain } conn = @@ -240,16 +230,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-8.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer == "facebook.com/page" assert pageview.referrer_source == "Facebook" end - test "ignores event when referrer is a spammer", %{conn: conn} do + test "ignores event when referrer is a spammer", %{conn: conn, domain: domain} do params = %{ - domain: "ignore-spammers-test.com", + domain: domain, name: "pageview", url: "http://gigride.live/", referrer: "https://www.1-best-seo.com", @@ -262,16 +252,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) assert response(conn, 202) == "ok" - assert !get_event("ignore-spammers-test.com") + assert !get_event(domain) end - test "feature flag - blocks traffic from a domain when block_traffic is enabled", %{ + test "blocks traffic from a domain when it's blocked", %{ conn: conn } do - FunWithFlags.enable(:block_event_ingest, for_actor: "feature-flag-test.com") + site = insert(:site, ingest_rate_limit_threshold: 0) params = %{ - domain: "feature-flag-test.com", + domain: site.domain, name: "pageview", url: "https://feature-flag-test.com" } @@ -282,15 +272,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) assert response(conn, 202) == "ok" - refute get_event("feature-flag-test.com") + refute get_event(site.domain) end - test "ignores when referrer is internal", %{conn: conn} do + test "ignores when referrer is internal", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https://gigride.live", - domain: "external-controller-test-9.com" + domain: domain } conn = @@ -298,18 +288,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-9.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "" end - test "ignores localhost referrer", %{conn: conn} do + test "ignores localhost referrer", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "http://localhost:4000/", - domain: "external-controller-test-10.com" + domain: domain } conn = @@ -317,18 +307,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-10.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "" end - test "parses subdomain referrer", %{conn: conn} do + test "parses subdomain referrer", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https://blog.gigride.live", - domain: "external-controller-test-11.com" + domain: domain } conn = @@ -336,68 +326,68 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-11.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "blog.gigride.live" end - test "referrer is cleaned", %{conn: conn} do + test "referrer is cleaned", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/", referrer: "https://www.indiehackers.com/page?query=param#hash", - domain: "external-controller-test-12.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-12.com") + pageview = get_event(domain) assert pageview.referrer == "indiehackers.com/page" end - test "utm_source overrides referrer source", %{conn: conn} do + test "utm_source overrides referrer source", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/?utm_source=betalist", referrer: "https://betalist.com/my-produxct", - domain: "external-controller-test-13.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-13.com") + pageview = get_event(domain) assert pageview.referrer_source == "betalist" end - test "utm tags are stored", %{conn: conn} do + test "utm tags are stored", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/?utm_medium=ads&utm_source=instagram&utm_campaign=video_story", - domain: "external-controller-test-utm-tags.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-utm-tags.com") + pageview = get_event(domain) assert pageview.utm_medium == "ads" assert pageview.utm_source == "instagram" assert pageview.utm_campaign == "video_story" end - test "if it's an :unknown referrer, just the domain is used", %{conn: conn} do + test "if it's an :unknown referrer, just the domain is used", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https://www.indiehackers.com/landing-page-feedback", - domain: "external-controller-test-14.com" + domain: domain } conn = @@ -405,18 +395,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-14.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "indiehackers.com" end - test "if the referrer is not http or https, it is ignored", %{conn: conn} do + test "if the referrer is not http or https, it is ignored", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "android-app://com.google.android.gm", - domain: "external-controller-test-15.com" + domain: domain } conn = @@ -424,18 +414,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-15.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.referrer_source == "" end - test "screen size is calculated from screen_width", %{conn: conn} do + test "screen size is calculated from screen_width", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", screen_width: 480, - domain: "external-controller-test-16.com" + domain: domain } conn = @@ -443,17 +433,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-16.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.screen_size == "Mobile" end - test "screen size is nil if screen_width is missing", %{conn: conn} do + test "screen size is nil if screen_width is missing", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", - domain: "external-controller-test-17.com" + domain: domain } conn = @@ -461,17 +451,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event("external-controller-test-17.com") + pageview = get_event(domain) assert response(conn, 202) == "ok" assert pageview.screen_size == "" end - test "can trigger a custom event", %{conn: conn} do + test "can trigger a custom event", %{conn: conn, domain: domain} do params = %{ name: "custom event", url: "http://gigride.live/", - domain: "external-controller-test-18.com" + domain: domain } conn = @@ -479,17 +469,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - event = get_event("external-controller-test-18.com") + event = get_event(domain) assert response(conn, 202) == "ok" assert event.name == "custom event" end - test "casts custom props to string", %{conn: conn} do + test "casts custom props to string", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test.com", + domain: domain, props: %{ bool_test: true, number_test: 12 @@ -499,51 +489,51 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - event = get_event("custom-prop-test.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["bool_test", "number_test"] assert Map.get(event, :"meta.value") == ["true", "12"] end - test "ignores malformed custom props", %{conn: conn} do + test "ignores malformed custom props", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-2.com", + domain: domain, props: "\"show-more:button\"" } conn |> post("/api/event", params) - event = get_event("custom-prop-test-2.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == [] assert Map.get(event, :"meta.value") == [] end - test "can send props stringified", %{conn: conn} do + test "can send props stringified", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-3.com", + domain: domain, props: Jason.encode!(%{number_test: 12}) } conn |> post("/api/event", params) - event = get_event("custom-prop-test-3.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["number_test"] assert Map.get(event, :"meta.value") == ["12"] end - test "ignores custom prop with array value", %{conn: conn} do + test "ignores custom prop with array value", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-4.com", + domain: domain, props: Jason.encode!(%{wat: ["some-thing"], other: "key"}) } @@ -551,17 +541,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 202 - event = get_event("custom-prop-test-4.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["other"] assert Map.get(event, :"meta.value") == ["key"] end - test "ignores custom prop with map value", %{conn: conn} do + test "ignores custom prop with map value", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-5.com", + domain: domain, props: Jason.encode!(%{foo: %{bar: "baz"}, other_key: 1}) } @@ -569,17 +559,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 202 - event = get_event("custom-prop-test-5.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["other_key"] assert Map.get(event, :"meta.value") == ["1"] end - test "ignores custom prop with empty string value", %{conn: conn} do + test "ignores custom prop with empty string value", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-empty-string-val.com", + domain: domain, props: Jason.encode!(%{foo: "", other_key: true}) } @@ -587,17 +577,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 202 - event = get_event("custom-prop-test-empty-string-val.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["other_key"] assert Map.get(event, :"meta.value") == ["true"] end - test "ignores custom prop with nil value", %{conn: conn} do + test "ignores custom prop with nil value", %{conn: conn, domain: domain} do params = %{ name: "Signup", url: "http://gigride.live/", - domain: "custom-prop-test-nil.com", + domain: domain, props: Jason.encode!(%{foo: nil, other_key: true}) } @@ -605,18 +595,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 202 - event = get_event("custom-prop-test-nil.com") + event = get_event(domain) assert Map.get(event, :"meta.key") == ["other_key"] assert Map.get(event, :"meta.value") == ["true"] end - test "ignores a malformed referrer URL", %{conn: conn} do + test "ignores a malformed referrer URL", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://gigride.live/", referrer: "https:://twitter.com", - domain: "external-controller-test-19.com" + domain: domain } conn = @@ -624,17 +614,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - event = get_event("external-controller-test-19.com") + event = get_event(domain) assert response(conn, 202) == "ok" assert event.referrer == "" end # Fake data is set up in config/test.exs - test "looks up location data from the ip address", %{conn: conn} do + test "looks up location data from the ip address", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-20.com", + domain: domain, url: "http://gigride.live/" } @@ -642,7 +632,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "2.2.2.2") |> post("/api/event", params) - pageview = get_event("external-controller-test-20.com") + pageview = get_event(domain) assert pageview.country_code == "FR" assert pageview.subdivision1_code == "FR-IDF" @@ -650,10 +640,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.city_geoname_id == 2_988_507 end - test "ignores unknown country code ZZ", %{conn: conn} do + test "ignores unknown country code ZZ", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-zz-country.com", + domain: domain, url: "http://gigride.live/" } @@ -661,15 +651,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "0.0.0.0") |> post("/api/event", params) - pageview = get_event("external-controller-test-zz-country.com") + pageview = get_event(domain) assert pageview.country_code == <<0, 0>> end - test "scrubs port from x-forwarded-for", %{conn: conn} do + test "scrubs port from x-forwarded-for", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-x-forwarded-for-port.com", + domain: domain, url: "http://gigride.live/" } @@ -677,15 +667,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "1.1.1.1:123") |> post("/api/event", params) - pageview = get_event("external-controller-test-x-forwarded-for-port.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "works with ipv6 without port in x-forwarded-for", %{conn: conn} do + test "works with ipv6 without port in x-forwarded-for", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-x-forwarded-for-ipv6.com", + domain: domain, url: "http://gigride.live/" } @@ -693,15 +683,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "1:1:1:1:1:1:1:1") |> post("/api/event", params) - pageview = get_event("external-controller-test-x-forwarded-for-ipv6.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn} do + test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-x-forwarded-for-ipv6-port.com", + domain: domain, url: "http://gigride.live/" } @@ -709,15 +699,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "[1:1:1:1:1:1:1:1]:123") |> post("/api/event", params) - pageview = get_event("external-controller-test-x-forwarded-for-ipv6-port.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "uses cloudflare's special header for client IP address if present", %{conn: conn} do + test "uses cloudflare's special header for client IP address if present", %{ + conn: conn, + domain: domain + } do params = %{ name: "pageview", - domain: "external-controller-test-cloudflare.com", + domain: domain, url: "http://gigride.live/" } @@ -726,15 +719,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("cf-connecting-ip", "1.1.1.1") |> post("/api/event", params) - pageview = get_event("external-controller-test-cloudflare.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "uses BunnyCDN's custom header for client IP address if present", %{conn: conn} do + test "uses BunnyCDN's custom header for client IP address if present", %{ + conn: conn, + domain: domain + } do params = %{ name: "pageview", - domain: "external-controller-test-bunny.com", + domain: domain, url: "http://gigride.live/" } @@ -743,17 +739,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("b-forwarded-for", "1.1.1.1,9.9.9.9") |> post("/api/event", params) - pageview = get_event("external-controller-test-bunny.com") + pageview = get_event(domain) assert pageview.country_code == "US" end test "Uses the Forwarded header when cf-connecting-ip and x-forwarded-for are missing", %{ - conn: conn + conn: conn, + domain: domain } do params = %{ name: "pageview", - domain: "external-controller-test-forwarded.com", + domain: domain, url: "http://gigride.live/" } @@ -761,15 +758,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("forwarded", "by=0.0.0.0;for=1.1.1.1;host=somehost.com;proto=https") |> post("/api/event", params) - pageview = get_event("external-controller-test-forwarded.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "Forwarded header can parse ipv6", %{conn: conn} do + test "Forwarded header can parse ipv6", %{conn: conn, domain: domain} do params = %{ name: "pageview", - domain: "external-controller-test-forwarded-ipv6.com", + domain: domain, url: "http://gigride.live/" } @@ -780,32 +777,32 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do ) |> post("/api/event", params) - pageview = get_event("external-controller-test-forwarded-ipv6.com") + pageview = get_event(domain) assert pageview.country_code == "US" end - test "URL is decoded", %{conn: conn} do + test "URL is decoded", %{conn: conn, domain: domain} do params = %{ name: "pageview", url: "http://www.example.com/opportunity/category/%D8%AC%D9%88%D8%A7%D8%A6%D8%B2-%D9%88%D9%85%D8%B3%D8%A7%D8%A8%D9%82%D8%A7%D8%AA", - domain: "external-controller-test-21.com" + domain: domain } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-21.com") + pageview = get_event(domain) assert pageview.pathname == "/opportunity/category/جوائز-ومسابقات" end - test "accepts shorthand map keys", %{conn: conn} do + test "accepts shorthand map keys", %{conn: conn, domain: domain} do params = %{ n: "pageview", u: "http://www.example.com/opportunity", - d: "external-controller-test-22.com", + d: domain, r: "https://facebook.com/page", w: 300 } @@ -813,7 +810,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - pageview = get_event("external-controller-test-22.com") + pageview = get_event(domain) assert pageview.pathname == "/opportunity" assert pageview.referrer_source == "Facebook" @@ -821,78 +818,78 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.screen_size == "Mobile" end - test "records hash when in hash mode", %{conn: conn} do + test "records hash when in hash mode", %{conn: conn, domain: domain} do params = %{ n: "pageview", u: "http://www.example.com/#page-a", - d: "external-controller-test-23.com", + d: domain, h: 1 } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-23.com") + pageview = get_event(domain) assert pageview.pathname == "/#page-a" end - test "does not record hash when hash mode is 0", %{conn: conn} do + test "does not record hash when hash mode is 0", %{conn: conn, domain: domain} do params = %{ n: "pageview", u: "http://www.example.com/#page-a", - d: "external-controller-test-hash-0.com", + d: domain, h: 0 } conn |> post("/api/event", params) - pageview = get_event("external-controller-test-hash-0.com") + pageview = get_event(domain) assert pageview.pathname == "/" end - test "decodes URL pathname, fragment and search", %{conn: conn} do + test "decodes URL pathname, fragment and search", %{conn: conn, domain: domain} do params = %{ n: "pageview", u: "https://test.com/%EF%BA%9D%EF%BB%AD%EF%BA%8E%EF%BA%8B%EF%BA%AF-%EF%BB%AE%EF%BB%A4%EF%BA%B3%EF%BA%8E%EF%BA%92%EF%BB%97%EF%BA%8E%EF%BA%97?utm_source=%25balle%25", - d: "url-decode-test.com", + d: domain, h: 1 } conn |> post("/api/event", params) - pageview = get_event("url-decode-test.com") + pageview = get_event(domain) assert pageview.hostname == "test.com" assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ" assert pageview.utm_source == "%balle%" end - test "can use double quotes in query params", %{conn: conn} do + test "can use double quotes in query params", %{conn: conn, domain: domain} do q = URI.encode_query(%{"utm_source" => "Something \"quoted\""}) params = %{ n: "pageview", u: "https://test.com/?" <> q, - d: "quote-encode-test.com", + d: domain, h: 1 } conn |> post("/api/event", params) - pageview = get_event("quote-encode-test.com") + pageview = get_event(domain) assert pageview.utm_source == "Something \"quoted\"" end - test "responds 400 when required fields are missing", %{conn: conn} do + test "responds 400 when required fields are missing", %{conn: conn, domain: domain} do params = %{ - domain: "some-domain.com", + domain: domain, name: "pageview" } @@ -903,7 +900,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert json_response(conn, 400) == %{ "errors" => %{ - "hostname" => ["can't be blank"] + "url" => ["is required"] } } end @@ -922,19 +919,25 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert json_response(conn, 400) == %{ "errors" => %{ - "domain" => ["can't be blank"], - "hostname" => ["can't be blank"], - "user_id" => ["can't be blank"] + "domain" => ["can't be blank"] } } end end describe "user_id generation" do - test "with same IP address and user agent, the same user ID is generated", %{conn: conn} do + setup do + site1 = insert(:site) + {:ok, domain: site1.domain} + end + + test "with same IP address and user agent, the same user ID is generated", %{ + conn: conn, + domain: domain + } do params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain.com", + domain: domain, name: "pageview" } @@ -948,15 +951,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "127.0.0.1") |> post("/api/event", params) - [one, two] = get_events("user-id-test-domain.com") + [one, two] = get_events(domain) assert one.user_id == two.user_id end - test "different IP address results in different user ID", %{conn: conn} do + test "different IP address results in different user ID", %{conn: conn, domain: domain} do params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain-2.com", + domain: domain, name: "pageview" } @@ -970,15 +973,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "982.32.12.1") |> post("/api/event", params) - [one, two] = get_events("user-id-test-domain-2.com") + [one, two] = get_events(domain) assert one.user_id != two.user_id end - test "different user agent results in different user ID", %{conn: conn} do + test "different user agent results in different user ID", %{conn: conn, domain: domain} do params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain-3.com", + domain: domain, name: "pageview" } @@ -992,15 +995,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "127.0.0.1") |> post("/api/event", params) - [one, two] = get_events("user-id-test-domain-3.com") + [one, two] = get_events(domain) assert one.user_id != two.user_id end - test "different domain value results in different user ID", %{conn: conn} do + test "different domain value results in different user ID", %{conn: conn, domain: domain1} do + domain2 = insert(:site).domain + params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain-4.com", + domain: domain1, name: "pageview" } @@ -1012,18 +1017,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("user-agent", @user_agent) |> put_req_header("x-forwarded-for", "127.0.0.1") - |> post("/api/event", Map.put(params, :domain, "other-domain.com")) + |> post("/api/event", Map.put(params, :domain, domain2)) - one = get_event("user-id-test-domain-4.com") - two = get_event("other-domain.com") + one = get_event(domain1) + two = get_event(domain2) assert one.user_id != two.user_id end - test "different hostname results in different user ID", %{conn: conn} do + test "different hostname results in different user ID", %{conn: conn, domain: domain} do params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain-5.com", + domain: domain, name: "pageview" } @@ -1037,17 +1042,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "127.0.0.1") |> post("/api/event", Map.put(params, :url, "https://other-domain.com/")) - [one, two] = get_events("user-id-test-domain-5.com") + [one, two] = get_events(domain) assert one.user_id != two.user_id end test "different hostname results in the same user ID when the root domain in the same", %{ - conn: conn + conn: conn, + domain: domain } do params = %{ url: "https://user-id-test-domain.com/", - domain: "user-id-test-domain-6.com", + domain: domain, name: "pageview" } @@ -1061,46 +1067,53 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "127.0.0.1") |> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/")) - [one, two] = get_events("user-id-test-domain-6.com") + [one, two] = get_events(domain) assert one.user_id == two.user_id end end - test "defaults hostname to (none) when missing", %{conn: conn} do - params = %{ - domain: "url-with-hostname-missing.com", - name: "pageview", - url: "file:///android_asset/www/index.html" - } + describe "remaining" do + setup do + site1 = insert(:site) + {:ok, domain: site1.domain} + end - conn = - conn - |> put_req_header("content-type", "text/plain") - |> post("/api/event", Jason.encode!(params)) + test "defaults hostname to (none) when missing", %{conn: conn, domain: domain} do + params = %{ + domain: domain, + name: "pageview", + url: "file:///android_asset/www/index.html" + } - pageview = get_event("url-with-hostname-missing.com") + conn = + conn + |> put_req_header("content-type", "text/plain") + |> post("/api/event", Jason.encode!(params)) - assert response(conn, 202) == "ok" - assert pageview.hostname == "(none)" - end + pageview = get_event(domain) - test "accepts chrome extension URLs", %{conn: conn} do - params = %{ - domain: "chrome-extension-url.com", - name: "pageview", - url: "chrome-extension://liipgellkffekalgefpjolodblggkmjg/popup.html" - } + assert response(conn, 202) == "ok" + assert pageview.hostname == "(none)" + end - conn = - conn - |> put_req_header("content-type", "text/plain") - |> post("/api/event", Jason.encode!(params)) + test "accepts chrome extension URLs", %{conn: conn, domain: domain} do + params = %{ + domain: domain, + name: "pageview", + url: "chrome-extension://liipgellkffekalgefpjolodblggkmjg/popup.html" + } - pageview = get_event("chrome-extension-url.com") + conn = + conn + |> put_req_header("content-type", "text/plain") + |> post("/api/event", Jason.encode!(params)) + + pageview = get_event(domain) - assert response(conn, 202) == "ok" - assert pageview.hostname == "liipgellkffekalgefpjolodblggkmjg" + assert response(conn, 202) == "ok" + assert pageview.hostname == "liipgellkffekalgefpjolodblggkmjg" + end end describe "GET /api/health" do @@ -1110,4 +1123,24 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert conn.status == 200 end end + + defp get_event(domain) do + Plausible.Event.WriteBuffer.flush() + + ClickhouseRepo.one( + from e in Plausible.ClickhouseEvent, + where: e.domain == ^domain, + order_by: [desc: e.timestamp] + ) + end + + defp get_events(domain) do + Plausible.Event.WriteBuffer.flush() + + ClickhouseRepo.all( + from e in Plausible.ClickhouseEvent, + where: e.domain == ^domain, + order_by: [desc: e.timestamp] + ) + end end