diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index ee3214276..b91f41f9b 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -9,7 +9,8 @@ defmodule Plausible.Ingestion.Event do request is identified as spam, or blocked. """ def build_and_buffer(%Request{} = request) do - with :ok <- spam_or_blocked?(request), + with :ok <- skip_if_spam(request), + :ok <- skip_if_blocked(request), salts <- Plausible.Session.Salts.fetch(), event <- Map.new(), %{} = event <- put_user_agent(event, request), @@ -18,11 +19,11 @@ defmodule Plausible.Ingestion.Event do %{} = event <- put_geolocation(event, request), %{} = event <- put_screen_size(event, request), %{} = event <- put_props(event, request), - events when is_list(events) <- map_domains(event, request), - events when is_list(events) <- put_user_id(events, request, salts), - {:ok, events} <- validate_events(events), - events when is_list(events) <- register_session(events, request, salts) do - Enum.each(events, &Plausible.Event.WriteBuffer.insert/1) + %{} = event <- put_user_id(event, request, salts), + {:ok, event} <- validate_event(event), + %{} = event <- register_session(event, request, salts) do + Plausible.Event.WriteBuffer.insert(event) + :ok end end @@ -35,6 +36,7 @@ defmodule Plausible.Ingestion.Event do end event + |> Map.put(:domain, request.domain) |> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)) |> Map.put(:name, request.event_name) |> Map.put(:hostname, Request.sanitize_hostname(host)) @@ -249,38 +251,20 @@ defmodule Plausible.Ingestion.Event do defp ignore_unknown_country("ZZ"), do: nil defp ignore_unknown_country(country), do: country - defp map_domains(%{} = event, %Request{} = request) do - domains = - if request.domain do - String.split(request.domain, ",") - |> Enum.map(&String.trim/1) - |> Enum.map(&strip_www/1) - else - uri = request.url && URI.parse(request.url) - [strip_www(uri && uri.host)] - end - - for domain <- domains, do: Map.put(event, :domain, domain) - end - - defp put_user_id(events, %Request{} = request, salts) do - for %{} = event <- events do - user_id = generate_user_id(request, event.domain, event.hostname, salts.current) - Map.put(event, :user_id, user_id) - end + defp put_user_id(%{} = event, %Request{} = request, salts) do + user_id = generate_user_id(request, event.domain, event.hostname, salts.current) + Map.put(event, :user_id, user_id) end - defp register_session(events, %Request{} = request, salts) do - for %Plausible.ClickhouseEvent{} = event <- events do - previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous) + defp register_session(event, %Request{} = request, salts) do + previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous) - session_id = - Tracer.with_span "cache_store_event" do - Plausible.Session.CacheStore.on_event(event, previous_user_id) - end + session_id = + Tracer.with_span "cache_store_event" do + Plausible.Session.CacheStore.on_event(event, previous_user_id) + end - Map.put(event, :session_id, session_id) - end + Map.put(event, :session_id, session_id) end defp generate_user_id(request, domain, hostname, salt) do @@ -308,7 +292,18 @@ defmodule Plausible.Ingestion.Event do end end - defp spam_or_blocked?(%Request{} = request) do + defp skip_if_spam(%Request{} = request) do + if request.referrer && + URI.parse(request.referrer).host + |> Request.sanitize_hostname() + |> ReferrerBlocklist.is_spammer?() do + :skip + else + :ok + end + end + + defp skip_if_blocked(%Request{} = request) do cond do request.domain in Application.get_env(:plausible, :domain_blacklist) -> :skip @@ -317,32 +312,18 @@ defmodule Plausible.Ingestion.Event do Tracer.set_attribute("blocked_by_flag", true) :skip - request.referrer && - URI.parse(request.referrer).host |> strip_www() |> ReferrerBlocklist.is_spammer?() -> - :skip - true -> :ok end end - defp validate_events(events) do - Enum.reduce_while(events, {:ok, []}, fn %{} = attrs, {:ok, acc} -> - attrs - |> Plausible.ClickhouseEvent.new() - |> Ecto.Changeset.apply_action(nil) - |> case do - {:ok, event} -> {:cont, {:ok, [event | acc]}} - {:error, changeset} -> {:halt, {:error, changeset}} - end - end) - end - - defp strip_www(hostname) do - if hostname do - String.replace_prefix(hostname, "www.", "") - else - nil + defp validate_event(%{} = attrs) do + attrs + |> Plausible.ClickhouseEvent.new() + |> Ecto.Changeset.apply_action(nil) + |> case do + {:ok, event} -> {:ok, event} + {:error, changeset} -> {:error, changeset} end end end diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index ba054f3ad..ba8ecf733 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -29,9 +29,9 @@ defmodule Plausible.Ingestion.Request do query_params: map() } - @spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json} + @spec build(Plug.Conn.t()) :: {:ok, [t()]} | {:error, :invalid_json} @doc """ - Builds a %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. + Builds a list of %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. """ def build(%Plug.Conn{} = conn) do with {:ok, request_body} <- parse_body(conn) do @@ -41,6 +41,7 @@ defmodule Plausible.Ingestion.Request do |> put_user_agent(conn) |> put_request_params(request_body) |> put_query_params() + |> map_domains(request_body) |> then(&{:ok, &1}) end end @@ -65,13 +66,26 @@ defmodule Plausible.Ingestion.Request do request | event_name: request_body["n"] || request_body["name"], referrer: request_body["r"] || request_body["referrer"], - domain: request_body["d"] || request_body["domain"], screen_width: request_body["w"] || request_body["screen_width"], hash_mode: request_body["h"] || request_body["hashMode"], props: parse_props(request_body) } end + defp map_domains(%__MODULE__{} = request, %{} = request_body) do + domains = + if raw = request_body["d"] || request_body["domain"] do + raw + |> String.split(",") + |> Enum.map(&sanitize_hostname/1) + else + request.uri + |> sanitize_hostname() + |> List.wrap() + end + + for domain <- domains, do: %__MODULE__{request | domain: domain} + end defp put_uri(%__MODULE__{} = request, %{} = request_body) do if url = request_body["u"] || request_body["url"] do