From 7fcb6a8c9bf35a19c2919a7db79fd2f18a14ad3e Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Mon, 19 Feb 2024 15:26:59 +0200 Subject: [PATCH] Remove old columns from being accessible in elixir code --- lib/plausible/clickhouse_event_v2.ex | 56 ++++++++++------------------ test/support/factory.ex | 12 ++---- test/support/test_utils.ex | 46 ++++++++++++++++++----- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 7252f3410..3071f2539 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -15,25 +15,6 @@ defmodule Plausible.ClickhouseEventV2 do field :session_id, Ch, type: "UInt64" field :timestamp, :naive_datetime - field :referrer, :string - field :referrer_source, :string - field :utm_medium, :string - field :utm_source, :string - field :utm_campaign, :string - field :utm_content, :string - field :utm_term, :string - - field :country_code, Ch, type: "FixedString(2)" - field :subdivision1_code, Ch, type: "LowCardinality(String)" - field :subdivision2_code, Ch, type: "LowCardinality(String)" - field :city_geoname_id, Ch, type: "UInt32" - - field :screen_size, Ch, type: "LowCardinality(String)" - field :operating_system, Ch, type: "LowCardinality(String)" - field :operating_system_version, Ch, type: "LowCardinality(String)" - field :browser, Ch, type: "LowCardinality(String)" - field :browser_version, Ch, type: "LowCardinality(String)" - field :"meta.key", {:array, :string} field :"meta.value", {:array, :string} @@ -42,7 +23,26 @@ defmodule Plausible.ClickhouseEventV2 do field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))" field :revenue_reporting_currency, Ch, type: "FixedString(3)" - field :transferred_from, :string + # Fields which are in the schema but not managed by us anymore. + # field :referrer, :string + # field :referrer_source, :string + # field :utm_medium, :string + # field :utm_source, :string + # field :utm_campaign, :string + # field :utm_content, :string + # field :utm_term, :string + + # field :country_code, Ch, type: "FixedString(2)" + # field :subdivision1_code, Ch, type: "LowCardinality(String)" + # field :subdivision2_code, Ch, type: "LowCardinality(String)" + # field :city_geoname_id, Ch, type: "UInt32" + + # field :screen_size, Ch, type: "LowCardinality(String)" + # field :operating_system, Ch, type: "LowCardinality(String)" + # field :operating_system_version, Ch, type: "LowCardinality(String)" + # field :browser, Ch, type: "LowCardinality(String)" + # field :browser_version, Ch, type: "LowCardinality(String)" + # field :transferred_from, :string end def new(attrs) do @@ -56,22 +56,6 @@ defmodule Plausible.ClickhouseEventV2 do :pathname, :user_id, :timestamp, - :operating_system, - :operating_system_version, - :browser, - :browser_version, - :referrer, - :referrer_source, - :utm_medium, - :utm_source, - :utm_campaign, - :utm_content, - :utm_term, - :country_code, - :subdivision1_code, - :subdivision2_code, - :city_geoname_id, - :screen_size, :"meta.key", :"meta.value", :revenue_source_amount, diff --git a/test/support/factory.ex b/test/support/factory.ex index 7c1ced73e..a54853649 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -69,24 +69,20 @@ defmodule Plausible.Factory do end def pageview_factory do - struct!( - event_factory(), - %{ - name: "pageview" - } - ) + Map.put(event_factory(), :name, "pageview") end def event_factory do hostname = sequence(:domain, &"example-#{&1}.com") - %Plausible.ClickhouseEventV2{ + %{ hostname: hostname, site_id: Enum.random(1000..10_000), pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()) + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + _factory_event: true } end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index fa5f9e9a3..4b1177323 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -82,9 +82,8 @@ defmodule Plausible.TestUtils do |> Map.put(:site_id, pageview.site.id) Factory.build(:pageview, pageview) - |> Map.from_struct() - |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) + |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews) @@ -94,9 +93,8 @@ defmodule Plausible.TestUtils do events = Enum.map(events, fn event -> Factory.build(:event, event) - |> Map.from_struct() - |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) + |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events) @@ -150,7 +148,7 @@ defmodule Plausible.TestUtils do event = Map.put(event, :site_id, site.id) case event do - %Plausible.ClickhouseEventV2{} -> + %{_factory_event: true} -> event imported_event -> @@ -181,7 +179,7 @@ defmodule Plausible.TestUtils do end) |> Enum.split_with(fn event -> case event do - %Plausible.ClickhouseEventV2{} -> + %{_factory_event: true} -> true _ -> @@ -196,14 +194,42 @@ defmodule Plausible.TestUtils do defp populate_native_stats(events) do sessions = Enum.reduce(events, %{}, fn event, sessions -> - # :TODO: Handle session parameters separately - session_id = Plausible.Session.CacheStore.on_event(event, event, nil) + # :KLUDGE: Callsites should not use :referrer, etc directly and instead + # nest it under session: %{ referrer: "foo" }. This is left here + # for backwards compatibility / to avoid breaking too many PRs. + session_params = + event + |> Map.take([ + :referrer, + :referrer_source, + :utm_medium, + :utm_source, + :utm_campaign, + :utm_content, + :utm_term, + :country_code, + :subdivision1_code, + :subdivision2_code, + :city_geoname_id, + :screen_size, + :operating_system, + :operating_system_version, + :browser, + :browser_version + ]) + |> Map.merge(Map.get(event, :session, %{})) + + session_id = Plausible.Session.CacheStore.on_event(event, session_params, nil) Map.put(sessions, {event.site_id, event.user_id}, session_id) end) Enum.each(events, fn event -> - event = Map.put(event, :session_id, sessions[{event.site_id, event.user_id}]) - Plausible.Event.WriteBuffer.insert(event) + clickhouse_event = + %Plausible.ClickhouseEventV2{} + |> Map.merge(event) + |> Map.put(:session_id, sessions[{event.site_id, event.user_id}]) + + Plausible.Event.WriteBuffer.insert(clickhouse_event) end) Plausible.Session.WriteBuffer.flush()