Limit ingestion payload size (#2904)

This pull request adds constraints on user input values during ingestion. The purpose of this PR is to filter out unrealistic or invalid inputs. More comments about each limit inline.
pull/2943/head
Vini Brasil 4 years ago committed by GitHub
parent 00d05b2edd
commit 4ca2e26b9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -37,6 +37,7 @@ defmodule Plausible.Goal do
|> validate_event_name_and_page_path()
|> update_change(:event_name, &String.trim/1)
|> update_change(:page_path, &String.trim/1)
|> validate_length(:event_name, max: 120)
end
defp validate_event_name_and_page_path(changeset) do

@ -8,11 +8,13 @@ defmodule Plausible.Ingestion.Request do
use Ecto.Schema
alias Ecto.Changeset
@max_url_size 2_000
embedded_schema do
field :remote_ip, :string
field :user_agent, :string
field :event_name, :string
field :uri, :string
field :uri, :map
field :hostname, :string
field :referrer, :string
field :domains, {:array, :string}
@ -47,6 +49,8 @@ defmodule Plausible.Ingestion.Request do
|> put_hostname()
|> put_user_agent(conn)
|> put_request_params(request_body)
|> put_referrer(request_body)
|> put_props(request_body)
|> put_pathname()
|> put_query_params()
|> map_domains(request_body)
@ -56,7 +60,7 @@ defmodule Plausible.Ingestion.Request do
:pathname,
:timestamp
])
|> Changeset.validate_length(:pathname, max: 2000)
|> Changeset.validate_length(:event_name, max: 120)
|> Changeset.apply_action(nil)
{:error, :invalid_json} ->
@ -87,12 +91,21 @@ defmodule Plausible.Ingestion.Request do
Changeset.change(
changeset,
event_name: request_body["n"] || request_body["name"],
referrer: request_body["r"] || request_body["referrer"],
hash_mode: request_body["h"] || request_body["hashMode"],
props: parse_props(request_body)
hash_mode: request_body["h"] || request_body["hashMode"]
)
end
defp put_referrer(changeset, %{} = request_body) do
referrer = request_body["r"] || request_body["referrer"]
if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer)
else
changeset
end
end
defp put_pathname(changeset) do
uri = Changeset.get_field(changeset, :uri)
hash_mode = Changeset.get_field(changeset, :hash_mode)
@ -130,12 +143,13 @@ defmodule Plausible.Ingestion.Request do
@disallowed_schemes ~w(data)
defp put_uri(changeset, %{} = request_body) do
with url when is_binary(url) <- request_body["u"] || request_body["url"],
url when byte_size(url) <= @max_url_size <- url,
%URI{} = uri when uri.scheme not in @disallowed_schemes <- URI.parse(url) do
Changeset.put_change(changeset, :uri, uri)
else
nil -> Changeset.add_error(changeset, :url, "is required")
%URI{} -> Changeset.add_error(changeset, :url, "scheme is not allowed")
_ -> Changeset.add_error(changeset, :url, "must be a string")
_ -> Changeset.add_error(changeset, :url, "must be a valid url")
end
end
@ -149,42 +163,49 @@ defmodule Plausible.Ingestion.Request do
Changeset.put_change(changeset, :hostname, sanitize_hostname(host))
end
defp parse_props(%{} = request_body) do
raw_props =
request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]
defp put_props(changeset, %{} = request_body) do
props =
(request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"])
|> decode_props_or_fallback()
|> Enum.reject(fn {_k, v} -> is_nil(v) || is_list(v) || is_map(v) || v == "" end)
|> Map.new()
case decode_raw_props(raw_props) do
{:ok, parsed_json} ->
parsed_json
|> Enum.filter(&valid_prop_value?/1)
|> Map.new()
_error ->
%{}
end
changeset
|> Changeset.put_change(:props, props)
|> validate_props()
end
defp decode_raw_props(props) when is_map(props), do: {:ok, props}
defp decode_raw_props(raw_json) when is_binary(raw_json) do
case Jason.decode(raw_json) do
{:ok, parsed_props} when is_map(parsed_props) ->
{:ok, parsed_props}
_ ->
:not_a_map
defp decode_props_or_fallback(raw) do
with raw when is_binary(raw) <- raw,
{:ok, %{} = decoded} <- Jason.decode(raw) do
decoded
else
already_a_map when is_map(already_a_map) -> already_a_map
{:ok, _list_or_other} -> %{}
{:error, _decode_error} -> %{}
_any -> %{}
end
end
defp decode_raw_props(_), do: :bad_format
defp valid_prop_value?({key, value}) do
case {key, value} do
{_key, ""} -> false
{_key, nil} -> false
{_key, value} when is_list(value) -> false
{_key, value} when is_map(value) -> false
{_key, _value} -> true
@max_prop_key_length 300
@max_prop_value_length 2000
defp validate_props(changeset) do
case Changeset.get_field(changeset, :props) do
props ->
Enum.reduce_while(props, changeset, fn
{key, value}, changeset
when byte_size(key) > @max_prop_key_length or
byte_size(value) > @max_prop_value_length ->
{:halt,
Changeset.add_error(
changeset,
:props,
"keys should have at most #{@max_prop_key_length} bytes and values #{@max_prop_value_length} bytes"
)}
_, changeset ->
{:cont, changeset}
end)
end
end

@ -12,6 +12,12 @@ defmodule Plausible.GoalsTest do
assert goal.event_name == "some event name"
end
test "create/2 validates goal name is at most 120 chars" do
site = insert(:site)
assert {:error, changeset} = Goals.create(site, %{"event_name" => String.duplicate("a", 130)})
assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]
end
test "for_site2 returns trimmed input even if it was saved with trailing whitespace" do
site = insert(:site)
insert(:goal, %{site: site, event_name: " Signup "})

@ -213,18 +213,84 @@ defmodule Plausible.Ingestion.RequestTest do
assert {"scheme is not allowed", _} = changeset.errors[:url]
end
test "returns validation error when pathname is too long" do
long_string = for _ <- 1..5000, do: "a", into: ""
test "returns validation error when url is too long" do
payload = %{
name: "pageview",
domain: "dummy.site",
url: "https://dummy.site/#{String.duplicate("a", 5000)}"
}
conn = build_conn(:post, "/api/events", payload)
assert {:error, changeset} = Request.build(conn)
assert {"must be a valid url", _} = changeset.errors[:url]
end
test "returns validation error when event name is too long" do
payload = %{
name: String.duplicate("a", 500),
domain: "dummy.site",
url: "https://dummy.site/"
}
conn = build_conn(:post, "/api/events", payload)
assert {:error, changeset} = Request.build(conn)
assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]
end
test "truncates referrer when too long" do
payload = %{
name: "pageview",
domain: "dummy.site",
url: "https://dummy.site/#{long_string}"
url: "https://dummy.site/",
referrer: String.duplicate("a", 2500)
}
conn = build_conn(:post, "/api/events", payload)
assert {:ok, request} = Request.build(conn)
assert request.referrer == String.duplicate("a", 2000)
end
test "returns validation error when props keys are too long" do
payload = %{
name: "pageview",
domain: "dummy.site",
url: "https://dummy.site/",
props: %{String.duplicate("a", 500) => "abc"}
}
conn = build_conn(:post, "/api/events", payload)
assert {:error, changeset} = Request.build(conn)
assert {"keys should have at most 300 bytes and values 2000 bytes", _} =
changeset.errors[:props]
end
test "returns validation error when props values are too long" do
payload = %{
name: "pageview",
domain: "dummy.site",
url: "https://dummy.site/",
props: %{"abc" => String.duplicate("a", 2500)}
}
conn = build_conn(:post, "/api/events", payload)
assert {:error, changeset} = Request.build(conn)
assert {"should be at most %{count} character(s)", _} = changeset.errors[:pathname]
assert {"keys should have at most 300 bytes and values 2000 bytes", _} =
changeset.errors[:props]
end
test "does not fail when sending many props" do
payload = %{
name: "pageview",
domain: "dummy.site",
url: "https://dummy.site/",
props: for(i <- 1..100, do: {"key_#{i}", "value"}, into: %{})
}
conn = build_conn(:post, "/api/events", payload)
assert {:ok, request} = Request.build(conn)
assert map_size(request.props) == 100
end
test "malicious input, technically valid json" do

Loading…
Cancel
Save