From 875ddf68e8abb9d9e52cc2ac7f60071c3865859e Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Sun, 3 Mar 2024 20:50:18 +0100 Subject: [PATCH] satanic --- lib/plausible/ingestion/event.ex | 22 +++++++++-- test/plausible/funnels_test.exs | 63 ++++++++++++++++++++----------- test/support/journey.ex | 64 ++++++++++++++++++++++++-------- test/support/test_utils.ex | 7 ++++ 4 files changed, 114 insertions(+), 42 deletions(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 59c0765c5..61d96ea04 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -45,6 +45,8 @@ defmodule Plausible.Ingestion.Event do @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} def build_and_buffer(%Request{domains: domains} = request) do + # IO.inspect(request, label: request.remote_ip) + processed_events = if spam_referrer?(request) do for domain <- domains, do: drop(new(domain, request), :spam_referrer) @@ -118,10 +120,12 @@ defmodule Plausible.Ingestion.Event do defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event -> - case pipeline_step.(acc_event) do - %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} - %__MODULE__{dropped?: false} = event -> {:cont, event} - end + timeit(inspect(pipeline_step), fn -> + case pipeline_step.(acc_event) do + %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} + %__MODULE__{dropped?: false} = event -> {:cont, event} + end + end) end) end @@ -500,4 +504,14 @@ defmodule Plausible.Ingestion.Event do end defp spam_referrer?(_), do: false + + def timeit(label, f) do + {time, result} = :timer.tc(f) + + if time / 1_000 > 1 do + # IO.inspect(time / 1_000, label: label) + end + + result + end end diff --git a/test/plausible/funnels_test.exs b/test/plausible/funnels_test.exs index 99b9a3bca..f32a51b86 100644 --- a/test/plausible/funnels_test.exs +++ b/test/plausible/funnels_test.exs @@ -3,7 +3,7 @@ defmodule Plausible.FunnelsTest do @moduletag :full_build_only use Plausible - use Journey + use Plausible.Test.Support.Journey on_full_build do alias Plausible.Goals @@ -238,13 +238,16 @@ defmodule Plausible.FunnelsTest do [g1, g2, g3] ) - populate_stats(site, [ - build(:pageview, pathname: "/go/to/blog/foo", user_id: 123), - build(:event, name: "Signup", user_id: 123), - build(:pageview, pathname: "/checkout", user_id: 123), - build(:pageview, pathname: "/go/to/blog/bar", user_id: 666), - build(:event, name: "Signup", user_id: 666) - ]) + journey site do + pageview "/go/to/blog/foo" + custom_event "Signup" + pageview "/checkout" + end + + journey site do + pageview "/go/to/blog/foo" + custom_event "Signup" + end query = Plausible.Stats.Query.from(site, %{"period" => "all"}) @@ -336,28 +339,38 @@ defmodule Plausible.FunnelsTest do }} = funnel_data end - @tag :slow - test "sampling", %{site: site, steps: [g1, g2, g3 | _]} do + def rand_ua do + :crypto.strong_rand_bytes(10) |> Base.encode64() + end + + # @tag :slow + @tag timeout: :timer.minutes(2) + test "sampling", %{steps: [g1, g2, g3 | _]} do + site = insert(:site, domain: "1.1.1.1") + {:ok, funnel} = Funnels.create( site, - "From blog to signup and purchase", + "From blog to signup", [g1, g2, g3] ) - stats = - 1..50_000 - |> Enum.flat_map(fn _ -> - user_id = SipHash.hash!("0123456789ABCDEF", :crypto.strong_rand_bytes(64)) + journey site, + manual: SpamJourney, + ip: &random_ipv6/0, + user_agent: nil do + pageview "/go/to/blog/foo" + custom_event "Signup" + pageview "/checkout" + end - [ - build(:pageview, pathname: "/go/to/blog/foo", user_id: user_id), - build(:event, name: "Signup", user_id: user_id), - build(:pageview, pathname: "/checkout", user_id: user_id) - ] - end) + 1..50_000 + |> Task.async_stream(fn i -> + SpamJourney.run() + end) + |> Stream.run() - populate_stats(site, stats) + IO.inspect(:done) query = Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"}) @@ -367,4 +380,10 @@ defmodule Plausible.FunnelsTest do end end end + + def mytimeit(label, f) do + {time, result} = :timer.tc(f) + IO.inspect(time / 1_000, label: label) + result + end end diff --git a/test/support/journey.ex b/test/support/journey.ex index 106f3c178..9d1d46b49 100644 --- a/test/support/journey.ex +++ b/test/support/journey.ex @@ -1,8 +1,8 @@ -defmodule Journey do +defmodule Plausible.Test.Support.Journey do defmacro __using__(_) do quote do - require Journey - import Journey + require Plausible.Test.Support.Journey + import Plausible.Test.Support.Journey end end @@ -12,7 +12,12 @@ defmodule Journey do def run(site, state, journey) do Enum.reduce(journey, state, fn {:pageview, [url, opts]}, state -> - payload = %{name: "pageview", domain: site.domain, url: build_url(site.domain, url, opts)} + payload = %{ + name: "pageview", + domain: site.domain, + url: build_url(site.domain, url, opts) + } + payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1)) {:custom_event, [name, opts]}, state -> @@ -20,7 +25,9 @@ defmodule Journey do payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1)) end) - flush_buffers() + if !state[:manual] do + flush_buffers() + end end defp build_url(domain, url, _opts) do @@ -30,22 +37,34 @@ defmodule Journey do defp new_conn(payload, state) do (state.conn || build_conn(:post, "/api/events", payload)) |> put_req_header("content-type", "application/json") - |> put_req_header("x-forwarded-for", state.ip) - |> put_req_header("user-agent", state.user_agent) + |> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip)) + |> maybe_add_header("user-agent", invoke_if_function(state.user_agent)) end - defp eval_now(now) when is_function(now, 0), do: now.() - defp eval_now(%NaiveDateTime{} = now), do: now + defp maybe_add_header(conn, _header, nil) do + conn + end - defp flush_buffers do + defp maybe_add_header(conn, header, value) do + put_req_header(conn, header, value) + end + + defp invoke_if_function(f) when is_function(f, 0), do: f.() + defp invoke_if_function(value), do: value + + def flush_buffers do Plausible.Session.WriteBuffer.flush() Plausible.Event.WriteBuffer.flush() end defp ingest(conn, state, idle_offset) do - now = eval_now(state.now) - {:ok, request} = Plausible.Ingestion.Request.build(conn, now) + now = invoke_if_function(state.now) + + {:ok, request} = + Plausible.Ingestion.Request.build(conn, now) + Plausible.Ingestion.Event.build_and_buffer(request) + new_now = NaiveDateTime.add(now, idle_offset, :second) Map.put(state, :now, new_now) end @@ -68,9 +87,12 @@ defmodule Journey do defmacro journey(site, state \\ [], do: block) do mod = :"#{__CALLER__.module}.Journey#{:erlang.phash2(binding())}" - __journey__(mod, site, state, block) + __journey__(aliased(state[:manual]) || mod, site, state, block) end + defp aliased({:__aliases__, _, mod}), do: Module.concat(mod) + defp aliased(other), do: other + defp __journey__(mod, site, state, block) do quote do defmodule unquote(mod) do @@ -80,12 +102,22 @@ defmodule Journey do unquote(block) - def run() do - Journey.run(@site, @initial_state, Enum.reverse(@journey)) + def run(overrides \\ %{}) do + Plausible.Test.Support.Journey.run( + @site, + Map.merge(@initial_state, overrides), + Enum.reverse(@journey) + ) + end + + def flush() do + Plausible.Test.Support.Journey.flush_buffers() end end - unquote(mod).run() + if is_nil(unquote(state)[:manual]) do + unquote(mod).run() + end end end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 1890fc57b..d0b9a1635 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -280,4 +280,11 @@ defmodule Plausible.TestUtils do def random_ip() do Enum.map_join(1..4, ".", fn _ -> Enum.random(1..254) end) end + + def random_ipv6() do + <> = + :crypto.strong_rand_bytes(16) + + "#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}" + end end