diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index e3a9979fb..3dc69eb15 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -13,7 +13,6 @@ defmodule Plausible.Ingestion.Event do defstruct domain: nil, site: nil, clickhouse_event_attrs: %{}, - clickhouse_session_attrs: %{}, clickhouse_event: nil, dropped?: false, drop_reason: nil, @@ -34,7 +33,6 @@ defmodule Plausible.Ingestion.Event do domain: String.t() | nil, site: %Plausible.Site{} | nil, clickhouse_event_attrs: map(), - clickhouse_session_attrs: map(), clickhouse_event: %ClickhouseEventV2{} | nil, dropped?: boolean(), drop_reason: drop_reason(), @@ -65,8 +63,6 @@ defmodule Plausible.Ingestion.Event do @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} def build_and_buffer(%Request{domains: domains} = request) do - # IO.inspect(request, label: request.remote_ip) - processed_events = if spam_referrer?(request) do for domain <- domains, do: drop(new(domain, request), :spam_referrer) @@ -140,12 +136,10 @@ defmodule Plausible.Ingestion.Event do defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event -> - timeit(inspect(pipeline_step), fn -> - case pipeline_step.(acc_event) do - %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} - %__MODULE__{dropped?: false} = event -> {:cont, event} - end - end) + case pipeline_step.(acc_event) do + %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} + %__MODULE__{dropped?: false} = event -> {:cont, event} + end end) end @@ -167,14 +161,10 @@ defmodule Plausible.Ingestion.Event do struct!(event, fields) end - defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do + defp update_attrs(%__MODULE__{} = event, %{} = attrs) do struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) end - defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do - struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs)) - end - defp drop_datacenter_ip(%__MODULE__{} = event) do case event.request.ip_classification do "dc_ip" -> @@ -207,7 +197,7 @@ defmodule Plausible.Ingestion.Event do drop(event, :bot) %UAInspector.Result{} = user_agent -> - update_session_attrs(event, %{ + update_attrs(event, %{ operating_system: os_name(user_agent), operating_system_version: os_version(user_agent), browser: browser_name(user_agent), @@ -221,7 +211,7 @@ defmodule Plausible.Ingestion.Event do end defp put_basic_info(%__MODULE__{} = event) do - update_event_attrs(event, %{ + update_attrs(event, %{ domain: event.domain, site_id: event.site.id, timestamp: event.request.timestamp, @@ -234,7 +224,7 @@ defmodule Plausible.Ingestion.Event do defp put_referrer(%__MODULE__{} = event) do ref = parse_referrer(event.request.uri, event.request.referrer) - update_session_attrs(event, %{ + update_attrs(event, %{ referrer_source: get_referrer_source(event.request, ref), referrer: clean_referrer(ref) }) @@ -243,7 +233,7 @@ defmodule Plausible.Ingestion.Event do defp put_utm_tags(%__MODULE__{} = event) do query_params = event.request.query_params - update_session_attrs(event, %{ + update_attrs(event, %{ utm_medium: query_params["utm_medium"], utm_source: query_params["utm_source"], utm_campaign: query_params["utm_campaign"], @@ -255,16 +245,17 @@ defmodule Plausible.Ingestion.Event do defp put_geolocation(%__MODULE__{} = event) do case event.request.ip_classification do "anonymous_vpn_ip" -> - update_session_attrs(event, %{country_code: "A1"}) + result = %{country_code: "A1"} + update_attrs(event, result) _any -> result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{} - update_session_attrs(event, result) + update_attrs(event, result) end end defp drop_shield_rule_country( - %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event + %__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event ) when is_binary(domain) and is_binary(cc) do case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do @@ -282,7 +273,7 @@ defmodule Plausible.Ingestion.Event do # defensive: ensuring the keys/values are always in the same order {keys, values} = Enum.unzip(props) - update_event_attrs(event, %{ + update_attrs(event, %{ "meta.key": keys, "meta.value": values }) @@ -293,7 +284,7 @@ defmodule Plausible.Ingestion.Event do defp put_revenue(event) do on_full_build do attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event) - update_event_attrs(event, attrs) + update_attrs(event, attrs) else event end @@ -304,7 +295,7 @@ defmodule Plausible.Ingestion.Event do end defp put_user_id(%__MODULE__{} = event) do - update_event_attrs(event, %{ + update_attrs(event, %{ user_id: generate_user_id( event.request, @@ -527,14 +518,4 @@ defmodule Plausible.Ingestion.Event do end defp spam_referrer?(_), do: false - - def timeit(label, f) do - {time, result} = :timer.tc(f) - - if time / 1_000 > 1 do - # IO.inspect(time / 1_000, label: label) - end - - result - end end diff --git a/lib/plausible/shield/ip_rule_cache.ex b/lib/plausible/shield/ip_rule_cache.ex index 9b941ba19..b8ca1de06 100644 --- a/lib/plausible/shield/ip_rule_cache.ex +++ b/lib/plausible/shield/ip_rule_cache.ex @@ -38,6 +38,8 @@ defmodule Plausible.Shield.IPRuleCache do end @impl true + def get_from_source({_, "0.0.0.0"}), do: nil + def get_from_source({domain, address}) do query = base_db_query() diff --git a/test/plausible/funnels_test.exs b/test/plausible/funnels_test.exs index f32a51b86..01fb88b71 100644 --- a/test/plausible/funnels_test.exs +++ b/test/plausible/funnels_test.exs @@ -1,5 +1,5 @@ defmodule Plausible.FunnelsTest do - use Plausible.DataCase + use Plausible.DataCase, async: true @moduletag :full_build_only use Plausible @@ -338,52 +338,6 @@ defmodule Plausible.FunnelsTest do ] }} = funnel_data end - - def rand_ua do - :crypto.strong_rand_bytes(10) |> Base.encode64() - end - - # @tag :slow - @tag timeout: :timer.minutes(2) - test "sampling", %{steps: [g1, g2, g3 | _]} do - site = insert(:site, domain: "1.1.1.1") - - {:ok, funnel} = - Funnels.create( - site, - "From blog to signup", - [g1, g2, g3] - ) - - journey site, - manual: SpamJourney, - ip: &random_ipv6/0, - user_agent: nil do - pageview "/go/to/blog/foo" - custom_event "Signup" - pageview "/checkout" - end - - 1..50_000 - |> Task.async_stream(fn i -> - SpamJourney.run() - end) - |> Stream.run() - - IO.inspect(:done) - - query = - Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"}) - - {:ok, funnel_data} = Stats.funnel(site, query, funnel.id) - assert_in_delta funnel_data[:all_visitors], 50_000, 5000 - end end end - - def mytimeit(label, f) do - {time, result} = :timer.tc(f) - IO.inspect(time / 1_000, label: label) - result - end end diff --git a/test/support/journey.ex b/test/support/journey.ex index 9d1d46b49..ad6cdbdae 100644 --- a/test/support/journey.ex +++ b/test/support/journey.ex @@ -6,23 +6,30 @@ defmodule Plausible.Test.Support.Journey do end end - import Phoenix.ConnTest + import Plug.Adapters.Test.Conn, only: [conn: 4] import Plug.Conn def run(site, state, journey) do + conn = new_conn(state) + Enum.reduce(journey, state, fn {:pageview, [url, opts]}, state -> payload = %{ name: "pageview", domain: site.domain, - url: build_url(site.domain, url, opts) + url: build_url(site, url, opts) } - payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1)) + conn + |> Plug.Adapters.Test.Conn.conn(:post, "/api/events", payload) + |> ingest(state, Keyword.get(opts, :idle, 1)) {:custom_event, [name, opts]}, state -> - payload = %{name: name, domain: site.domain, url: build_url(site.domain, "/", opts)} - payload |> new_conn(state) |> ingest(state, Keyword.get(opts, :idle, 1)) + payload = %{name: name, domain: site.domain, url: build_url(site, "/", opts)} + + conn + |> conn(:post, "/api/events", payload) + |> ingest(state, Keyword.get(opts, :idle, 1)) end) if !state[:manual] do @@ -30,12 +37,26 @@ defmodule Plausible.Test.Support.Journey do end end - defp build_url(domain, url, _opts) do - "https://" <> Path.join(domain, url) + def build_url(site, path, params) do + if params[:url] do + params[:url] + else + site_domain = site.domain |> URI.encode_www_form() + + query_string = + params + |> Enum.into(%{}) + |> Map.take(~w[utm_source utm_medium utm_campaign utm_term utm_content]a) + |> URI.encode_query() + + uri = URI.new!("https://" <> Path.join(site_domain, path)) + uri = %{uri | query: query_string} + to_string(uri) + end end - defp new_conn(payload, state) do - (state.conn || build_conn(:post, "/api/events", payload)) + defp new_conn(state) do + (state.conn || %Plug.Conn{}) |> put_req_header("content-type", "application/json") |> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip)) |> maybe_add_header("user-agent", invoke_if_function(state.user_agent)) diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 5c99edbc6..ec0819cb8 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -262,6 +262,7 @@ defmodule Plausible.TestUtils do <> = :crypto.strong_rand_bytes(16) - "#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}:#{Base.encode16(<>, case: :lower)}" + [a1, a2, a3, a4, a5, a6, a7, a8] + |> Enum.map_join(":", &Base.encode16(<<&1>>, case: :lower)) end end