From c60a2faee4c79cde7005c918a231f81aec9bb326 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 6 Mar 2024 10:59:24 +0200 Subject: [PATCH] Write event table session columns (#3865) * Write event table session columns * Update testing factory rig --- lib/plausible/clickhouse_event_v2.ex | 35 +++++++++---------- lib/plausible/ingestion/event.ex | 29 +++++++++++++-- lib/plausible/session/cache_store.ex | 21 +++++------ test/plausible/ingestion/event_test.exs | 4 +-- .../api/external_controller_test.exs | 18 +++++++++- test/support/factory.ex | 5 ++- test/support/test_utils.ex | 29 +++++++-------- 7 files changed, 85 insertions(+), 56 deletions(-) diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 3071f2539..8792099df 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -23,26 +23,25 @@ defmodule Plausible.ClickhouseEventV2 do field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))" field :revenue_reporting_currency, Ch, type: "FixedString(3)" - # Fields which are in the schema but not managed by us anymore. - # field :referrer, :string - # field :referrer_source, :string - # field :utm_medium, :string - # field :utm_source, :string - # field :utm_campaign, :string - # field :utm_content, :string - # field :utm_term, :string + # Session attributes + field :referrer, :string + field :referrer_source, :string + field :utm_medium, :string + field :utm_source, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string - # field :country_code, Ch, type: "FixedString(2)" - # field :subdivision1_code, Ch, type: "LowCardinality(String)" - # field :subdivision2_code, Ch, type: "LowCardinality(String)" - # field :city_geoname_id, Ch, type: "UInt32" + field :country_code, Ch, type: "FixedString(2)" + field :subdivision1_code, Ch, type: "LowCardinality(String)" + field :subdivision2_code, Ch, type: "LowCardinality(String)" + field :city_geoname_id, Ch, type: "UInt32" - # field :screen_size, Ch, type: "LowCardinality(String)" - # field :operating_system, Ch, type: "LowCardinality(String)" - # field :operating_system_version, Ch, type: "LowCardinality(String)" - # field :browser, Ch, type: "LowCardinality(String)" - # field :browser_version, Ch, type: "LowCardinality(String)" - # field :transferred_from, :string + field :screen_size, Ch, type: "LowCardinality(String)" + field :operating_system, Ch, type: "LowCardinality(String)" + field :operating_system_version, Ch, type: "LowCardinality(String)" + field :browser, Ch, type: "LowCardinality(String)" + field :browser_version, Ch, type: "LowCardinality(String)" end def new(attrs) do diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 59c0765c5..f11c435ac 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -43,6 +43,26 @@ defmodule Plausible.Ingestion.Event do changeset: %Ecto.Changeset{} } + @session_properties [ + :session_id, + :referrer, + :referrer_source, + :utm_medium, + :utm_source, + :utm_campaign, + :utm_content, + :utm_term, + :country_code, + :subdivision1_code, + :subdivision2_code, + :city_geoname_id, + :screen_size, + :operating_system, + :operating_system_version, + :browser, + :browser_version + ] + @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} def build_and_buffer(%Request{domains: domains} = request) do processed_events = @@ -315,15 +335,18 @@ defmodule Plausible.Ingestion.Event do event.salts.previous ) - session_id = + session = Plausible.Session.CacheStore.on_event( event.clickhouse_event, event.clickhouse_session_attrs, previous_user_id ) - clickhouse_event = Map.put(event.clickhouse_event, :session_id, session_id) - %{event | clickhouse_event: clickhouse_event} + %{ + event + | clickhouse_event: + Map.merge(event.clickhouse_event, Map.take(session, @session_properties)) + } end defp write_to_buffer(%__MODULE__{clickhouse_event: clickhouse_event} = event) do diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index 565698b8c..6c29407d5 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -5,18 +5,15 @@ defmodule Plausible.Session.CacheStore do def on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) do found_session = find_session(event, event.user_id) || find_session(event, prev_user_id) - session = - if found_session do - updated_session = update_session(found_session, event) - buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}]) - persist_session(updated_session) - else - new_session = new_session_from_event(event, session_attributes) - buffer.insert([new_session]) - persist_session(new_session) - end - - session.session_id + if found_session do + updated_session = update_session(found_session, event) + buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}]) + persist_session(updated_session) + else + new_session = new_session_from_event(event, session_attributes) + buffer.insert([new_session]) + persist_session(new_session) + end end defp find_session(_domain, nil), do: nil diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index e2162dfcb..b6627d8bc 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -226,7 +226,7 @@ defmodule Plausible.Ingestion.EventTest do assert {:ok, request} = Request.build(conn) assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request) - assert event.clickhouse_event_attrs.hostname == "192.168.0.1" + assert event.clickhouse_event.hostname == "192.168.0.1" end test "Hostname is stored with public suffix processing" do @@ -241,6 +241,6 @@ defmodule Plausible.Ingestion.EventTest do assert {:ok, request} = Request.build(conn) assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request) - assert event.clickhouse_event_attrs.hostname == "foo.netlify.app" + assert event.clickhouse_event.hostname == "foo.netlify.app" end end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index 3583e3272..e37ec9e94 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -12,7 +12,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do {:ok, site: site} end - test "records the event", %{conn: conn, site: site} do + test "records the event and session", %{conn: conn, site: site} do params = %{ domain: site.domain, name: "pageview", @@ -26,6 +26,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" assert pageview.hostname == "example.com" @@ -33,6 +34,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.site_id == site.id assert pageview.pathname == "/" + assert pageview.session_id == session.session_id end test "works with Content-Type: text/plain", %{conn: conn, site: site} do @@ -224,12 +226,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) session = get_created_session(site) + event = get_event(site) assert response(conn, 202) == "ok" assert session.operating_system == "Mac" assert session.operating_system_version == "10.13" assert session.browser == "Chrome" assert session.browser_version == "70.0" + assert event.operating_system == "Mac" + assert event.operating_system_version == "10.13" + assert event.browser == "Chrome" + assert event.browser_version == "70.0" end test "parses referrer", %{conn: conn, site: site} do @@ -265,10 +272,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) session = get_created_session(site) + event = get_event(site) assert response(conn, 202) == "ok" assert session.referrer == "facebook.com/page" assert session.referrer_source == "Facebook" + assert event.referrer == session.referrer + assert event.referrer_source == session.referrer_source end test "ignores event when referrer is a spammer", %{conn: conn, site: site} do @@ -811,11 +821,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) session = get_created_session(site) + event = get_event(site) assert session.country_code == "GB" assert session.subdivision1_code == "GB-ENG" assert session.subdivision2_code == "GB-WBK" assert session.city_geoname_id == 2_655_045 + assert event.country_code == session.country_code + assert event.subdivision1_code == session.subdivision1_code + assert event.subdivision2_code == session.subdivision2_code + assert event.city_geoname_id == session.city_geoname_id end test "ignores unknown country code ZZ", %{conn: conn, site: site} do @@ -1108,6 +1123,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.hostname == "test.com" assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ" assert session.utm_source == "%balle%" + assert pageview.utm_source == session.utm_source end test "can use double quotes in query params", %{conn: conn, site: site} do diff --git a/test/support/factory.ex b/test/support/factory.ex index 090ca55d3..6ab32a750 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -89,14 +89,13 @@ defmodule Plausible.Factory do def event_factory do hostname = sequence(:domain, &"example-#{&1}.com") - %{ + %Plausible.ClickhouseEventV2{ hostname: hostname, site_id: Enum.random(1000..10_000), pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - _factory_event: true + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()) } end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 984731526..91db1b99b 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -82,8 +82,9 @@ defmodule Plausible.TestUtils do |> Map.put(:site_id, pageview.site.id) Factory.build(:pageview, pageview) + |> Map.from_struct() + |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) - |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews) @@ -93,8 +94,9 @@ defmodule Plausible.TestUtils do events = Enum.map(events, fn event -> Factory.build(:event, event) + |> Map.from_struct() + |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) - |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events) @@ -148,7 +150,7 @@ defmodule Plausible.TestUtils do event = Map.put(event, :site_id, site.id) case event do - %{_factory_event: true} -> + %Plausible.ClickhouseEventV2{} -> event imported_event -> @@ -179,7 +181,7 @@ defmodule Plausible.TestUtils do end) |> Enum.split_with(fn event -> case event do - %{_factory_event: true} -> + %Plausible.ClickhouseEventV2{} -> true _ -> @@ -192,20 +194,13 @@ defmodule Plausible.TestUtils do end defp populate_native_stats(events) do - sessions = - Enum.reduce(events, %{}, fn event, sessions -> - session_id = Plausible.Session.CacheStore.on_event(event, event, nil) - Map.put(sessions, {event.site_id, event.user_id}, session_id) - end) - - Enum.each(events, fn event -> - clickhouse_event = - %Plausible.ClickhouseEventV2{} - |> Map.merge(event) - |> Map.put(:session_id, sessions[{event.site_id, event.user_id}]) + for event_params <- events do + session = Plausible.Session.CacheStore.on_event(event_params, event_params, nil) - Plausible.Event.WriteBuffer.insert(clickhouse_event) - end) + event_params + |> Map.merge(session) + |> Plausible.Event.WriteBuffer.insert() + end Plausible.Session.WriteBuffer.flush() Plausible.Event.WriteBuffer.flush()