From f4d7ba70e8eb453accc3682f55eb2ed7df674c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cenk=20K=C3=BCc=C3=BCk?= Date: Fri, 17 Nov 2023 15:34:20 +0100 Subject: [PATCH] drop IPs classified as datacenter (#3522) * drop IPs classified as datacenter * process event and increment counter instead of dropping --- lib/plausible/ingestion/event.ex | 13 +++++++++++++ lib/plausible/ingestion/request.ex | 11 +++++++++++ test/plausible/ingestion/request_test.exs | 3 ++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 3955d685a..05924cb2b 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -24,6 +24,7 @@ defmodule Plausible.Ingestion.Event do | :spam_referrer | GateKeeper.policy() | :invalid + | :dc_ip @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -92,6 +93,7 @@ defmodule Plausible.Ingestion.Event do defp pipeline() do [ + &put_ip_classification/1, &put_user_agent/1, &put_basic_info/1, &put_referrer/1, @@ -138,6 +140,17 @@ defmodule Plausible.Ingestion.Event do struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) end + defp put_ip_classification(%__MODULE__{} = event) do + case event.request.ip_classification do + "dc_ip" -> + emit_telemetry_dropped(event, :dc_ip) + event + + _any -> + event + end + end + defp put_user_agent(%__MODULE__{} = event) do case parse_user_agent(event.request) do %UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}} -> diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index d220067a6..40c9e5155 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -36,6 +36,7 @@ defmodule Plausible.Ingestion.Request do field :hostname, :string field :referrer, :string field :domains, {:array, :string} + field :ip_classification, :string field :hash_mode, :integer field :pathname, :string field :props, :map @@ -63,6 +64,7 @@ defmodule Plausible.Ingestion.Request do case parse_body(conn) do {:ok, request_body} -> changeset + |> put_ip_classification(conn) |> put_remote_ip(conn) |> put_uri(request_body) |> put_hostname() @@ -272,6 +274,15 @@ defmodule Plausible.Ingestion.Request do end end + defp put_ip_classification(changeset, %Plug.Conn{} = conn) do + value = + conn + |> Plug.Conn.get_req_header("x-plausible-ip-type") + |> List.first() + + Changeset.put_change(changeset, :ip_classification, value) + end + defp put_user_agent(changeset, %Plug.Conn{} = conn) do user_agent = conn diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index 45f749389..ef1fb8d8e 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -457,7 +457,8 @@ defmodule Plausible.Ingestion.RequestTest do "remote_ip" => "127.0.0.1", "revenue_source" => %{"amount" => "12.3", "currency" => "USD"}, "uri" => "https://dummy.site/pictures/index.html?foo=bar&baz=bam", - "user_agent" => "Mozilla" + "user_agent" => "Mozilla", + "ip_classification" => nil } assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])