|
|
|
|
@ -13,7 +13,6 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
defstruct domain: nil,
|
|
|
|
|
site: nil,
|
|
|
|
|
clickhouse_event_attrs: %{},
|
|
|
|
|
clickhouse_session_attrs: %{},
|
|
|
|
|
clickhouse_event: nil,
|
|
|
|
|
dropped?: false,
|
|
|
|
|
drop_reason: nil,
|
|
|
|
|
@ -34,7 +33,6 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
domain: String.t() | nil,
|
|
|
|
|
site: %Plausible.Site{} | nil,
|
|
|
|
|
clickhouse_event_attrs: map(),
|
|
|
|
|
clickhouse_session_attrs: map(),
|
|
|
|
|
clickhouse_event: %ClickhouseEventV2{} | nil,
|
|
|
|
|
dropped?: boolean(),
|
|
|
|
|
drop_reason: drop_reason(),
|
|
|
|
|
@ -65,8 +63,6 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
|
|
|
|
|
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
|
|
|
|
|
def build_and_buffer(%Request{domains: domains} = request) do
|
|
|
|
|
# IO.inspect(request, label: request.remote_ip)
|
|
|
|
|
|
|
|
|
|
processed_events =
|
|
|
|
|
if spam_referrer?(request) do
|
|
|
|
|
for domain <- domains, do: drop(new(domain, request), :spam_referrer)
|
|
|
|
|
@ -140,12 +136,10 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
|
|
|
|
|
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
|
|
|
|
|
Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
|
|
|
|
|
timeit(inspect(pipeline_step), fn ->
|
|
|
|
|
case pipeline_step.(acc_event) do
|
|
|
|
|
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
|
|
|
|
|
%__MODULE__{dropped?: false} = event -> {:cont, event}
|
|
|
|
|
end
|
|
|
|
|
end)
|
|
|
|
|
case pipeline_step.(acc_event) do
|
|
|
|
|
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
|
|
|
|
|
%__MODULE__{dropped?: false} = event -> {:cont, event}
|
|
|
|
|
end
|
|
|
|
|
end)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
@ -167,14 +161,10 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
struct!(event, fields)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do
|
|
|
|
|
defp update_attrs(%__MODULE__{} = event, %{} = attrs) do
|
|
|
|
|
struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs))
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do
|
|
|
|
|
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp drop_datacenter_ip(%__MODULE__{} = event) do
|
|
|
|
|
case event.request.ip_classification do
|
|
|
|
|
"dc_ip" ->
|
|
|
|
|
@ -207,7 +197,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
drop(event, :bot)
|
|
|
|
|
|
|
|
|
|
%UAInspector.Result{} = user_agent ->
|
|
|
|
|
update_session_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
operating_system: os_name(user_agent),
|
|
|
|
|
operating_system_version: os_version(user_agent),
|
|
|
|
|
browser: browser_name(user_agent),
|
|
|
|
|
@ -221,7 +211,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp put_basic_info(%__MODULE__{} = event) do
|
|
|
|
|
update_event_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
domain: event.domain,
|
|
|
|
|
site_id: event.site.id,
|
|
|
|
|
timestamp: event.request.timestamp,
|
|
|
|
|
@ -234,7 +224,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
defp put_referrer(%__MODULE__{} = event) do
|
|
|
|
|
ref = parse_referrer(event.request.uri, event.request.referrer)
|
|
|
|
|
|
|
|
|
|
update_session_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
referrer_source: get_referrer_source(event.request, ref),
|
|
|
|
|
referrer: clean_referrer(ref)
|
|
|
|
|
})
|
|
|
|
|
@ -243,7 +233,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
defp put_utm_tags(%__MODULE__{} = event) do
|
|
|
|
|
query_params = event.request.query_params
|
|
|
|
|
|
|
|
|
|
update_session_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
utm_medium: query_params["utm_medium"],
|
|
|
|
|
utm_source: query_params["utm_source"],
|
|
|
|
|
utm_campaign: query_params["utm_campaign"],
|
|
|
|
|
@ -255,16 +245,17 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
defp put_geolocation(%__MODULE__{} = event) do
|
|
|
|
|
case event.request.ip_classification do
|
|
|
|
|
"anonymous_vpn_ip" ->
|
|
|
|
|
update_session_attrs(event, %{country_code: "A1"})
|
|
|
|
|
result = %{country_code: "A1"}
|
|
|
|
|
update_attrs(event, result)
|
|
|
|
|
|
|
|
|
|
_any ->
|
|
|
|
|
result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{}
|
|
|
|
|
update_session_attrs(event, result)
|
|
|
|
|
update_attrs(event, result)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp drop_shield_rule_country(
|
|
|
|
|
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
|
|
|
|
|
%__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event
|
|
|
|
|
)
|
|
|
|
|
when is_binary(domain) and is_binary(cc) do
|
|
|
|
|
case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
|
|
|
|
|
@ -282,7 +273,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
# defensive: ensuring the keys/values are always in the same order
|
|
|
|
|
{keys, values} = Enum.unzip(props)
|
|
|
|
|
|
|
|
|
|
update_event_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
"meta.key": keys,
|
|
|
|
|
"meta.value": values
|
|
|
|
|
})
|
|
|
|
|
@ -293,7 +284,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
defp put_revenue(event) do
|
|
|
|
|
on_full_build do
|
|
|
|
|
attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event)
|
|
|
|
|
update_event_attrs(event, attrs)
|
|
|
|
|
update_attrs(event, attrs)
|
|
|
|
|
else
|
|
|
|
|
event
|
|
|
|
|
end
|
|
|
|
|
@ -304,7 +295,7 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp put_user_id(%__MODULE__{} = event) do
|
|
|
|
|
update_event_attrs(event, %{
|
|
|
|
|
update_attrs(event, %{
|
|
|
|
|
user_id:
|
|
|
|
|
generate_user_id(
|
|
|
|
|
event.request,
|
|
|
|
|
@ -527,14 +518,4 @@ defmodule Plausible.Ingestion.Event do
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
defp spam_referrer?(_), do: false
|
|
|
|
|
|
|
|
|
|
def timeit(label, f) do
|
|
|
|
|
{time, result} = :timer.tc(f)
|
|
|
|
|
|
|
|
|
|
if time / 1_000 > 1 do
|
|
|
|
|
# IO.inspect(time / 1_000, label: label)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
result
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|