journey-rebased
Adam Rutkowski 3 years ago
parent 1bf2be88d3
commit 70063bc79e

@ -13,7 +13,6 @@ defmodule Plausible.Ingestion.Event do
defstruct domain: nil,
site: nil,
clickhouse_event_attrs: %{},
clickhouse_session_attrs: %{},
clickhouse_event: nil,
dropped?: false,
drop_reason: nil,
@ -34,7 +33,6 @@ defmodule Plausible.Ingestion.Event do
domain: String.t() | nil,
site: %Plausible.Site{} | nil,
clickhouse_event_attrs: map(),
clickhouse_session_attrs: map(),
clickhouse_event: %ClickhouseEventV2{} | nil,
dropped?: boolean(),
drop_reason: drop_reason(),
@ -65,8 +63,6 @@ defmodule Plausible.Ingestion.Event do
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do
# IO.inspect(request, label: request.remote_ip)
processed_events =
if spam_referrer?(request) do
for domain <- domains, do: drop(new(domain, request), :spam_referrer)
@ -140,12 +136,10 @@ defmodule Plausible.Ingestion.Event do
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
timeit(inspect(pipeline_step), fn ->
case pipeline_step.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end)
case pipeline_step.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end)
end
@ -167,14 +161,10 @@ defmodule Plausible.Ingestion.Event do
struct!(event, fields)
end
defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do
defp update_attrs(%__MODULE__{} = event, %{} = attrs) do
struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs))
end
defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
end
defp drop_datacenter_ip(%__MODULE__{} = event) do
case event.request.ip_classification do
"dc_ip" ->
@ -207,7 +197,7 @@ defmodule Plausible.Ingestion.Event do
drop(event, :bot)
%UAInspector.Result{} = user_agent ->
update_session_attrs(event, %{
update_attrs(event, %{
operating_system: os_name(user_agent),
operating_system_version: os_version(user_agent),
browser: browser_name(user_agent),
@ -221,7 +211,7 @@ defmodule Plausible.Ingestion.Event do
end
defp put_basic_info(%__MODULE__{} = event) do
update_event_attrs(event, %{
update_attrs(event, %{
domain: event.domain,
site_id: event.site.id,
timestamp: event.request.timestamp,
@ -234,7 +224,7 @@ defmodule Plausible.Ingestion.Event do
defp put_referrer(%__MODULE__{} = event) do
ref = parse_referrer(event.request.uri, event.request.referrer)
update_session_attrs(event, %{
update_attrs(event, %{
referrer_source: get_referrer_source(event.request, ref),
referrer: clean_referrer(ref)
})
@ -243,7 +233,7 @@ defmodule Plausible.Ingestion.Event do
defp put_utm_tags(%__MODULE__{} = event) do
query_params = event.request.query_params
update_session_attrs(event, %{
update_attrs(event, %{
utm_medium: query_params["utm_medium"],
utm_source: query_params["utm_source"],
utm_campaign: query_params["utm_campaign"],
@ -255,16 +245,17 @@ defmodule Plausible.Ingestion.Event do
defp put_geolocation(%__MODULE__{} = event) do
case event.request.ip_classification do
"anonymous_vpn_ip" ->
update_session_attrs(event, %{country_code: "A1"})
result = %{country_code: "A1"}
update_attrs(event, result)
_any ->
result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{}
update_session_attrs(event, result)
update_attrs(event, result)
end
end
defp drop_shield_rule_country(
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
%__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event
)
when is_binary(domain) and is_binary(cc) do
case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
@ -282,7 +273,7 @@ defmodule Plausible.Ingestion.Event do
# defensive: ensuring the keys/values are always in the same order
{keys, values} = Enum.unzip(props)
update_event_attrs(event, %{
update_attrs(event, %{
"meta.key": keys,
"meta.value": values
})
@ -293,7 +284,7 @@ defmodule Plausible.Ingestion.Event do
defp put_revenue(event) do
on_full_build do
attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event)
update_event_attrs(event, attrs)
update_attrs(event, attrs)
else
event
end
@ -304,7 +295,7 @@ defmodule Plausible.Ingestion.Event do
end
defp put_user_id(%__MODULE__{} = event) do
update_event_attrs(event, %{
update_attrs(event, %{
user_id:
generate_user_id(
event.request,
@ -527,14 +518,4 @@ defmodule Plausible.Ingestion.Event do
end
defp spam_referrer?(_), do: false
def timeit(label, f) do
{time, result} = :timer.tc(f)
if time / 1_000 > 1 do
# IO.inspect(time / 1_000, label: label)
end
result
end
end

@ -38,6 +38,8 @@ defmodule Plausible.Shield.IPRuleCache do
end
@impl true
def get_from_source({_, "0.0.0.0"}), do: nil
def get_from_source({domain, address}) do
query =
base_db_query()

@ -1,5 +1,5 @@
defmodule Plausible.FunnelsTest do
use Plausible.DataCase
use Plausible.DataCase, async: true
@moduletag :full_build_only
use Plausible
@ -338,52 +338,6 @@ defmodule Plausible.FunnelsTest do
]
}} = funnel_data
end
def rand_ua do
:crypto.strong_rand_bytes(10) |> Base.encode64()
end
# @tag :slow
@tag timeout: :timer.minutes(2)
test "sampling", %{steps: [g1, g2, g3 | _]} do
site = insert(:site, domain: "1.1.1.1")
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup",
[g1, g2, g3]
)
journey site,
manual: SpamJourney,
ip: &random_ipv6/0,
user_agent: nil do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
1..50_000
|> Task.async_stream(fn i ->
SpamJourney.run()
end)
|> Stream.run()
IO.inspect(:done)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
{:ok, funnel_data} = Stats.funnel(site, query, funnel.id)
assert_in_delta funnel_data[:all_visitors], 50_000, 5000
end
end
end
def mytimeit(label, f) do
{time, result} = :timer.tc(f)
IO.inspect(time / 1_000, label: label)
result
end
end

@ -6,23 +6,30 @@ defmodule Plausible.Test.Support.Journey do
end
end
import Phoenix.ConnTest
import Plug.Adapters.Test.Conn, only: [conn: 4]
import Plug.Conn
def run(site, state, journey) do
conn = new_conn(state)
Enum.reduce(journey, state, fn
{:pageview, [url, opts]}, state ->
payload = %{
name: "pageview",
domain: site.domain,
url: build_url(site.domain, url, opts)
url: build_url(site, url, opts)
}
payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1))
conn
|> Plug.Adapters.Test.Conn.conn(:post, "/api/events", payload)
|> ingest(state, Keyword.get(opts, :idle, 1))
{:custom_event, [name, opts]}, state ->
payload = %{name: name, domain: site.domain, url: build_url(site.domain, "/", opts)}
payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1))
payload = %{name: name, domain: site.domain, url: build_url(site, "/", opts)}
conn
|> conn(:post, "/api/events", payload)
|> ingest(state, Keyword.get(opts, :idle, 1))
end)
if !state[:manual] do
@ -30,12 +37,26 @@ defmodule Plausible.Test.Support.Journey do
end
end
defp build_url(domain, url, _opts) do
"https://" <> Path.join(domain, url)
def build_url(site, path, params) do
if params[:url] do
params[:url]
else
site_domain = site.domain |> URI.encode_www_form()
query_string =
params
|> Enum.into(%{})
|> Map.take(~w[utm_source utm_medium utm_campaign utm_term utm_content]a)
|> URI.encode_query()
uri = URI.new!("https://" <> Path.join(site_domain, path))
uri = %{uri | query: query_string}
to_string(uri)
end
end
defp new_conn(payload, state) do
(state.conn || build_conn(:post, "/api/events", payload))
defp new_conn(state) do
(state.conn || %Plug.Conn{})
|> put_req_header("content-type", "application/json")
|> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip))
|> maybe_add_header("user-agent", invoke_if_function(state.user_agent))

@ -262,6 +262,7 @@ defmodule Plausible.TestUtils do
<<a1::16, a2::16, a3::16, a4::16, a5::16, a6::16, a7::16, a8::16>> =
:crypto.strong_rand_bytes(16)
"#{Base.encode16(<<a1>>, case: :lower)}:#{Base.encode16(<<a2>>, case: :lower)}:#{Base.encode16(<<a3>>, case: :lower)}:#{Base.encode16(<<a4>>, case: :lower)}:#{Base.encode16(<<a5>>, case: :lower)}:#{Base.encode16(<<a6>>, case: :lower)}:#{Base.encode16(<<a7>>, case: :lower)}:#{Base.encode16(<<a8>>, case: :lower)}"
[a1, a2, a3, a4, a5, a6, a7, a8]
|> Enum.map_join(":", &Base.encode16(<<&1>>, case: :lower))
end
end

Loading…
Cancel
Save