journey
Adam Rutkowski 3 years ago
parent 60b9ca7d16
commit 875ddf68e8

@ -45,6 +45,8 @@ defmodule Plausible.Ingestion.Event do
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do
# IO.inspect(request, label: request.remote_ip)
processed_events =
if spam_referrer?(request) do
for domain <- domains, do: drop(new(domain, request), :spam_referrer)
@ -118,10 +120,12 @@ defmodule Plausible.Ingestion.Event do
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
case pipeline_step.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
timeit(inspect(pipeline_step), fn ->
case pipeline_step.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end)
end)
end
@ -500,4 +504,14 @@ defmodule Plausible.Ingestion.Event do
end
defp spam_referrer?(_), do: false
def timeit(label, f) do
{time, result} = :timer.tc(f)
if time / 1_000 > 1 do
# IO.inspect(time / 1_000, label: label)
end
result
end
end

@ -3,7 +3,7 @@ defmodule Plausible.FunnelsTest do
@moduletag :full_build_only
use Plausible
use Journey
use Plausible.Test.Support.Journey
on_full_build do
alias Plausible.Goals
@ -238,13 +238,16 @@ defmodule Plausible.FunnelsTest do
[g1, g2, g3]
)
populate_stats(site, [
build(:pageview, pathname: "/go/to/blog/foo", user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, pathname: "/checkout", user_id: 123),
build(:pageview, pathname: "/go/to/blog/bar", user_id: 666),
build(:event, name: "Signup", user_id: 666)
])
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
end
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
@ -336,28 +339,38 @@ defmodule Plausible.FunnelsTest do
}} = funnel_data
end
@tag :slow
test "sampling", %{site: site, steps: [g1, g2, g3 | _]} do
def rand_ua do
:crypto.strong_rand_bytes(10) |> Base.encode64()
end
# @tag :slow
@tag timeout: :timer.minutes(2)
test "sampling", %{steps: [g1, g2, g3 | _]} do
site = insert(:site, domain: "1.1.1.1")
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup and purchase",
"From blog to signup",
[g1, g2, g3]
)
stats =
1..50_000
|> Enum.flat_map(fn _ ->
user_id = SipHash.hash!("0123456789ABCDEF", :crypto.strong_rand_bytes(64))
journey site,
manual: SpamJourney,
ip: &random_ipv6/0,
user_agent: nil do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
[
build(:pageview, pathname: "/go/to/blog/foo", user_id: user_id),
build(:event, name: "Signup", user_id: user_id),
build(:pageview, pathname: "/checkout", user_id: user_id)
]
end)
1..50_000
|> Task.async_stream(fn i ->
SpamJourney.run()
end)
|> Stream.run()
populate_stats(site, stats)
IO.inspect(:done)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
@ -367,4 +380,10 @@ defmodule Plausible.FunnelsTest do
end
end
end
def mytimeit(label, f) do
{time, result} = :timer.tc(f)
IO.inspect(time / 1_000, label: label)
result
end
end

@ -1,8 +1,8 @@
defmodule Journey do
defmodule Plausible.Test.Support.Journey do
defmacro __using__(_) do
quote do
require Journey
import Journey
require Plausible.Test.Support.Journey
import Plausible.Test.Support.Journey
end
end
@ -12,7 +12,12 @@ defmodule Journey do
def run(site, state, journey) do
Enum.reduce(journey, state, fn
{:pageview, [url, opts]}, state ->
payload = %{name: "pageview", domain: site.domain, url: build_url(site.domain, url, opts)}
payload = %{
name: "pageview",
domain: site.domain,
url: build_url(site.domain, url, opts)
}
payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1))
{:custom_event, [name, opts]}, state ->
@ -20,7 +25,9 @@ defmodule Journey do
payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1))
end)
flush_buffers()
if !state[:manual] do
flush_buffers()
end
end
defp build_url(domain, url, _opts) do
@ -30,22 +37,34 @@ defmodule Journey do
defp new_conn(payload, state) do
(state.conn || build_conn(:post, "/api/events", payload))
|> put_req_header("content-type", "application/json")
|> put_req_header("x-forwarded-for", state.ip)
|> put_req_header("user-agent", state.user_agent)
|> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip))
|> maybe_add_header("user-agent", invoke_if_function(state.user_agent))
end
defp eval_now(now) when is_function(now, 0), do: now.()
defp eval_now(%NaiveDateTime{} = now), do: now
defp maybe_add_header(conn, _header, nil) do
conn
end
defp flush_buffers do
defp maybe_add_header(conn, header, value) do
put_req_header(conn, header, value)
end
defp invoke_if_function(f) when is_function(f, 0), do: f.()
defp invoke_if_function(value), do: value
def flush_buffers do
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()
end
defp ingest(conn, state, idle_offset) do
now = eval_now(state.now)
{:ok, request} = Plausible.Ingestion.Request.build(conn, now)
now = invoke_if_function(state.now)
{:ok, request} =
Plausible.Ingestion.Request.build(conn, now)
Plausible.Ingestion.Event.build_and_buffer(request)
new_now = NaiveDateTime.add(now, idle_offset, :second)
Map.put(state, :now, new_now)
end
@ -68,9 +87,12 @@ defmodule Journey do
defmacro journey(site, state \\ [], do: block) do
mod = :"#{__CALLER__.module}.Journey#{:erlang.phash2(binding())}"
__journey__(mod, site, state, block)
__journey__(aliased(state[:manual]) || mod, site, state, block)
end
defp aliased({:__aliases__, _, mod}), do: Module.concat(mod)
defp aliased(other), do: other
defp __journey__(mod, site, state, block) do
quote do
defmodule unquote(mod) do
@ -80,12 +102,22 @@ defmodule Journey do
unquote(block)
def run() do
Journey.run(@site, @initial_state, Enum.reverse(@journey))
def run(overrides \\ %{}) do
Plausible.Test.Support.Journey.run(
@site,
Map.merge(@initial_state, overrides),
Enum.reverse(@journey)
)
end
def flush() do
Plausible.Test.Support.Journey.flush_buffers()
end
end
unquote(mod).run()
if is_nil(unquote(state)[:manual]) do
unquote(mod).run()
end
end
end

@ -280,4 +280,11 @@ defmodule Plausible.TestUtils do
def random_ip() do
Enum.map_join(1..4, ".", fn _ -> Enum.random(1..254) end)
end
def random_ipv6() do
<<a1::16, a2::16, a3::16, a4::16, a5::16, a6::16, a7::16, a8::16>> =
:crypto.strong_rand_bytes(16)
"#{Base.encode16(<<a1>>, case: :lower)}:#{Base.encode16(<<a2>>, case: :lower)}:#{Base.encode16(<<a3>>, case: :lower)}:#{Base.encode16(<<a4>>, case: :lower)}:#{Base.encode16(<<a5>>, case: :lower)}:#{Base.encode16(<<a6>>, case: :lower)}:#{Base.encode16(<<a7>>, case: :lower)}:#{Base.encode16(<<a8>>, case: :lower)}"
end
end

Loading…
Cancel
Save