You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
analytics/lib/plausible/ingestion/request.ex

315 lines
8.8 KiB
Elixir

defmodule Plausible.Ecto.EventName do
@moduledoc """
Custom type for event name. Accepts Strings and Integers and stores them as String. Returns
cast error if any other type is provided. Accepting integers is important for 404 tracking.
"""
use Ecto.Type
def type, do: :string
def cast(val) when is_binary(val), do: {:ok, val}
def cast(val) when is_integer(val), do: {:ok, Integer.to_string(val)}
def cast(_), do: :error
def load(val), do: {:ok, val}
def dump(val), do: {:ok, val}
end
defmodule Plausible.Ingestion.Request do
@moduledoc """
The %Plausible.Ingestion.Request{} struct stores all needed fields
to create an event downstream. Pre-eliminary validation is made
to detect user errors early.
"""
use Ecto.Schema
alias Ecto.Changeset
@max_url_size 2_000
embedded_schema do
field :remote_ip, :string
field :user_agent, :string
field :event_name, Plausible.Ecto.EventName
field :uri, :map
field :hostname, :string
field :referrer, :string
field :domains, {:array, :string}
field :hash_mode, :integer
field :pathname, :string
field :props, :map
field :revenue_source, :map
field :query_params, :map
field :timestamp, :naive_datetime
end
@type t() :: %__MODULE__{}
@spec build(Plug.Conn.t(), NaiveDateTime.t()) :: {:ok, t()} | {:error, Changeset.t()}
@doc """
Builds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
"""
def build(%Plug.Conn{} = conn, now \\ NaiveDateTime.utc_now()) do
changeset =
%__MODULE__{}
|> Changeset.change()
|> Changeset.put_change(
:timestamp,
NaiveDateTime.truncate(now, :second)
)
case parse_body(conn) do
{:ok, request_body} ->
changeset
|> put_remote_ip(conn)
|> put_uri(request_body)
|> put_hostname()
|> put_user_agent(conn)
|> put_request_params(request_body)
|> put_referrer(request_body)
|> put_props(request_body)
|> put_pathname()
|> put_query_params()
|> put_revenue_source(request_body)
|> map_domains(request_body)
|> Changeset.validate_required([
:event_name,
:hostname,
:pathname,
:timestamp
])
|> Changeset.validate_length(:event_name, max: 120)
|> Changeset.apply_action(nil)
{:error, :invalid_json} ->
{:error, Changeset.add_error(changeset, :request, "Unable to parse request body as json")}
end
end
defp put_remote_ip(changeset, conn) do
Changeset.put_change(changeset, :remote_ip, PlausibleWeb.RemoteIp.get(conn))
end
defp parse_body(conn) do
case conn.body_params do
%Plug.Conn.Unfetched{} ->
{:ok, body, _conn} = Plug.Conn.read_body(conn)
case Jason.decode(body) do
{:ok, params} when is_map(params) -> {:ok, params}
_ -> {:error, :invalid_json}
end
params ->
{:ok, params}
end
end
defp put_request_params(changeset, %{} = request_body) do
Changeset.cast(
changeset,
%{
event_name: request_body["n"] || request_body["name"],
hash_mode: request_body["h"] || request_body["hashMode"]
},
[:event_name, :hash_mode]
)
end
defp put_referrer(changeset, %{} = request_body) do
referrer = request_body["r"] || request_body["referrer"]
if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer)
else
changeset
end
end
defp put_pathname(changeset) do
uri = Changeset.get_field(changeset, :uri)
hash_mode = Changeset.get_field(changeset, :hash_mode)
pathname = get_pathname(uri, hash_mode)
Changeset.put_change(changeset, :pathname, pathname)
end
defp map_domains(changeset, %{} = request_body) do
raw = request_body["d"] || request_body["domain"]
raw = if is_binary(raw), do: String.trim(raw)
case raw do
"" ->
Changeset.add_error(changeset, :domain, "can't be blank")
raw when is_binary(raw) ->
domains =
raw
|> String.split(",")
|> Enum.map(&sanitize_hostname/1)
Changeset.put_change(changeset, :domains, domains)
nil ->
from_uri = sanitize_hostname(Changeset.get_field(changeset, :uri))
if from_uri do
Changeset.put_change(changeset, :domains, [from_uri])
else
Changeset.add_error(changeset, :domain, "can't be blank")
end
end
end
@disallowed_schemes ~w(data)
defp put_uri(changeset, %{} = request_body) do
with url when is_binary(url) <- request_body["u"] || request_body["url"],
url when byte_size(url) <= @max_url_size <- url,
%URI{} = uri when uri.scheme not in @disallowed_schemes <- URI.parse(url) do
Changeset.put_change(changeset, :uri, uri)
else
nil -> Changeset.add_error(changeset, :url, "is required")
%URI{} -> Changeset.add_error(changeset, :url, "scheme is not allowed")
_ -> Changeset.add_error(changeset, :url, "must be a valid url")
end
end
defp put_hostname(changeset) do
host =
case Changeset.get_field(changeset, :uri) do
%{host: host} when is_binary(host) and host != "" -> host
_ -> "(none)"
end
Changeset.put_change(changeset, :hostname, sanitize_hostname(host))
end
@max_props 30
defp put_props(changeset, %{} = request_body) do
props =
(request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"])
|> decode_json_or_fallback()
|> Enum.reject(fn {_k, v} -> is_nil(v) || is_list(v) || is_map(v) || v == "" end)
|> Enum.take(@max_props)
|> Map.new()
changeset
|> Changeset.put_change(:props, props)
|> validate_props()
end
@max_prop_key_length 300
@max_prop_value_length 2000
defp validate_props(changeset) do
case Changeset.get_field(changeset, :props) do
props ->
Enum.reduce_while(props, changeset, fn
{key, value}, changeset
when byte_size(key) > @max_prop_key_length or
byte_size(value) > @max_prop_value_length ->
{:halt,
Changeset.add_error(
changeset,
:props,
"keys should have at most #{@max_prop_key_length} bytes and values #{@max_prop_value_length} bytes"
)}
_, changeset ->
{:cont, changeset}
end)
end
end
defp put_revenue_source(%Ecto.Changeset{} = changeset, %{} = request_body) do
with revenue_source <- request_body["revenue"] || request_body["$"],
%{"amount" => _, "currency" => _} = revenue_source <-
decode_json_or_fallback(revenue_source) do
parse_revenue_source(changeset, revenue_source)
else
_any -> changeset
end
end
@valid_currencies Plausible.Goal.valid_currencies()
defp parse_revenue_source(changeset, %{"amount" => amount, "currency" => currency}) do
with true <- currency in @valid_currencies,
{%Decimal{} = amount, _rest} <- parse_decimal(amount),
%Money{} = amount <- Money.new(currency, amount) do
Changeset.put_change(changeset, :revenue_source, amount)
else
_any -> changeset
end
end
defp decode_json_or_fallback(raw) do
with raw when is_binary(raw) <- raw,
{:ok, %{} = decoded} <- Jason.decode(raw) do
decoded
else
already_a_map when is_map(already_a_map) -> already_a_map
_any -> %{}
end
end
defp parse_decimal(value) do
case value do
value when is_binary(value) -> Decimal.parse(value)
value when is_float(value) -> {Decimal.from_float(value), nil}
value when is_integer(value) -> {Decimal.new(value), nil}
_any -> :error
end
end
defp put_query_params(changeset) do
case Changeset.get_field(changeset, :uri) do
%{query: query} when is_binary(query) ->
Changeset.put_change(changeset, :query_params, URI.decode_query(query))
_any ->
changeset
end
end
defp put_user_agent(changeset, %Plug.Conn{} = conn) do
user_agent =
conn
|> Plug.Conn.get_req_header("user-agent")
|> List.first()
Changeset.put_change(changeset, :user_agent, user_agent)
end
defp get_pathname(nil, _hash_mode), do: "/"
defp get_pathname(uri, hash_mode) do
pathname =
(uri.path || "/")
|> URI.decode()
|> String.trim_trailing()
if hash_mode == 1 && uri.fragment do
pathname <> "#" <> URI.decode(uri.fragment)
else
pathname
end
end
@doc """
Removes the "www" part of a hostname.
"""
def sanitize_hostname(%URI{host: hostname}) do
sanitize_hostname(hostname)
end
def sanitize_hostname(hostname) when is_binary(hostname) do
hostname
|> String.trim()
|> String.replace_prefix("www.", "")
end
def sanitize_hostname(nil) do
nil
end
end