Parse event domain in Plausible.Ingestion.Request

process-per-ip
Vinicius Brasil 4 years ago
parent 6ad9ccfaf3
commit d3d2868e4a

@ -9,7 +9,8 @@ defmodule Plausible.Ingestion.Event do
request is identified as spam, or blocked.
"""
def build_and_buffer(%Request{} = request) do
with :ok <- spam_or_blocked?(request),
with :ok <- skip_if_spam(request),
:ok <- skip_if_blocked(request),
salts <- Plausible.Session.Salts.fetch(),
event <- Map.new(),
%{} = event <- put_user_agent(event, request),
@ -18,11 +19,11 @@ defmodule Plausible.Ingestion.Event do
%{} = event <- put_geolocation(event, request),
%{} = event <- put_screen_size(event, request),
%{} = event <- put_props(event, request),
events when is_list(events) <- map_domains(event, request),
events when is_list(events) <- put_user_id(events, request, salts),
{:ok, events} <- validate_events(events),
events when is_list(events) <- register_session(events, request, salts) do
Enum.each(events, &Plausible.Event.WriteBuffer.insert/1)
%{} = event <- put_user_id(event, request, salts),
{:ok, event} <- validate_event(event),
%{} = event <- register_session(event, request, salts) do
Plausible.Event.WriteBuffer.insert(event)
:ok
end
end
@ -35,6 +36,7 @@ defmodule Plausible.Ingestion.Event do
end
event
|> Map.put(:domain, request.domain)
|> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))
|> Map.put(:name, request.event_name)
|> Map.put(:hostname, Request.sanitize_hostname(host))
@ -249,38 +251,20 @@ defmodule Plausible.Ingestion.Event do
defp ignore_unknown_country("ZZ"), do: nil
defp ignore_unknown_country(country), do: country
defp map_domains(%{} = event, %Request{} = request) do
domains =
if request.domain do
String.split(request.domain, ",")
|> Enum.map(&String.trim/1)
|> Enum.map(&strip_www/1)
else
uri = request.url && URI.parse(request.url)
[strip_www(uri && uri.host)]
end
for domain <- domains, do: Map.put(event, :domain, domain)
end
defp put_user_id(events, %Request{} = request, salts) do
for %{} = event <- events do
user_id = generate_user_id(request, event.domain, event.hostname, salts.current)
Map.put(event, :user_id, user_id)
end
defp put_user_id(%{} = event, %Request{} = request, salts) do
user_id = generate_user_id(request, event.domain, event.hostname, salts.current)
Map.put(event, :user_id, user_id)
end
defp register_session(events, %Request{} = request, salts) do
for %Plausible.ClickhouseEvent{} = event <- events do
previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous)
defp register_session(event, %Request{} = request, salts) do
previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous)
session_id =
Tracer.with_span "cache_store_event" do
Plausible.Session.CacheStore.on_event(event, previous_user_id)
end
session_id =
Tracer.with_span "cache_store_event" do
Plausible.Session.CacheStore.on_event(event, previous_user_id)
end
Map.put(event, :session_id, session_id)
end
Map.put(event, :session_id, session_id)
end
defp generate_user_id(request, domain, hostname, salt) do
@ -308,7 +292,18 @@ defmodule Plausible.Ingestion.Event do
end
end
defp spam_or_blocked?(%Request{} = request) do
defp skip_if_spam(%Request{} = request) do
if request.referrer &&
URI.parse(request.referrer).host
|> Request.sanitize_hostname()
|> ReferrerBlocklist.is_spammer?() do
:skip
else
:ok
end
end
defp skip_if_blocked(%Request{} = request) do
cond do
request.domain in Application.get_env(:plausible, :domain_blacklist) ->
:skip
@ -317,32 +312,18 @@ defmodule Plausible.Ingestion.Event do
Tracer.set_attribute("blocked_by_flag", true)
:skip
request.referrer &&
URI.parse(request.referrer).host |> strip_www() |> ReferrerBlocklist.is_spammer?() ->
:skip
true ->
:ok
end
end
defp validate_events(events) do
Enum.reduce_while(events, {:ok, []}, fn %{} = attrs, {:ok, acc} ->
attrs
|> Plausible.ClickhouseEvent.new()
|> Ecto.Changeset.apply_action(nil)
|> case do
{:ok, event} -> {:cont, {:ok, [event | acc]}}
{:error, changeset} -> {:halt, {:error, changeset}}
end
end)
end
defp strip_www(hostname) do
if hostname do
String.replace_prefix(hostname, "www.", "")
else
nil
defp validate_event(%{} = attrs) do
attrs
|> Plausible.ClickhouseEvent.new()
|> Ecto.Changeset.apply_action(nil)
|> case do
{:ok, event} -> {:ok, event}
{:error, changeset} -> {:error, changeset}
end
end
end

@ -29,9 +29,9 @@ defmodule Plausible.Ingestion.Request do
query_params: map()
}
@spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json}
@spec build(Plug.Conn.t()) :: {:ok, [t()]} | {:error, :invalid_json}
@doc """
Builds a %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
Builds a list of %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
"""
def build(%Plug.Conn{} = conn) do
with {:ok, request_body} <- parse_body(conn) do
@ -41,6 +41,7 @@ defmodule Plausible.Ingestion.Request do
|> put_user_agent(conn)
|> put_request_params(request_body)
|> put_query_params()
|> map_domains(request_body)
|> then(&{:ok, &1})
end
end
@ -65,13 +66,26 @@ defmodule Plausible.Ingestion.Request do
request
| event_name: request_body["n"] || request_body["name"],
referrer: request_body["r"] || request_body["referrer"],
domain: request_body["d"] || request_body["domain"],
screen_width: request_body["w"] || request_body["screen_width"],
hash_mode: request_body["h"] || request_body["hashMode"],
props: parse_props(request_body)
}
end
defp map_domains(%__MODULE__{} = request, %{} = request_body) do
domains =
if raw = request_body["d"] || request_body["domain"] do
raw
|> String.split(",")
|> Enum.map(&sanitize_hostname/1)
else
request.uri
|> sanitize_hostname()
|> List.wrap()
end
for domain <- domains, do: %__MODULE__{request | domain: domain}
end
defp put_uri(%__MODULE__{} = request, %{} = request_body) do
if url = request_body["u"] || request_body["url"] do

Loading…
Cancel
Save