diff --git a/extra/lib/plausible/ingestion/event/revenue.ex b/extra/lib/plausible/ingestion/event/revenue.ex index 94c4a3158..f05180622 100644 --- a/extra/lib/plausible/ingestion/event/revenue.ex +++ b/extra/lib/plausible/ingestion/event/revenue.ex @@ -7,7 +7,10 @@ defmodule Plausible.Ingestion.Event.Revenue do %Plausible.Ingestion.Event{request: %{revenue_source: %Money{} = revenue_source}} = event ) do matching_goal = - Enum.find(event.site.revenue_goals, &(&1.event_name == event.clickhouse_event_attrs.name)) + Enum.find( + event.site.revenue_goals, + &(&1.event_name == event.clickhouse_event.changes[:name]) + ) cond do is_nil(matching_goal) -> diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 3071f2539..74427bc1b 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -24,27 +24,62 @@ defmodule Plausible.ClickhouseEventV2 do field :revenue_reporting_currency, Ch, type: "FixedString(3)" # Fields which are in the schema but not managed by us anymore. - # field :referrer, :string - # field :referrer_source, :string - # field :utm_medium, :string - # field :utm_source, :string - # field :utm_campaign, :string - # field :utm_content, :string - # field :utm_term, :string + field :session_referrer, :string, virtual: true + field :session_referrer_source, :string, virtual: true + field :session_utm_medium, :string, virtual: true + field :session_utm_source, :string, virtual: true + field :session_utm_campaign, :string, virtual: true + field :session_utm_content, :string, virtual: true + field :session_utm_term, :string, virtual: true - # field :country_code, Ch, type: "FixedString(2)" - # field :subdivision1_code, Ch, type: "LowCardinality(String)" - # field :subdivision2_code, Ch, type: "LowCardinality(String)" - # field :city_geoname_id, Ch, type: "UInt32" + field :session_country_code, Ch, type: "FixedString(2)", virtual: true + field :session_subdivision1_code, Ch, type: "LowCardinality(String)", virtual: true + field :session_subdivision2_code, Ch, type: "LowCardinality(String)", virtual: true + field :session_city_geoname_id, Ch, type: "UInt32", virtual: true - # field :screen_size, Ch, type: "LowCardinality(String)" - # field :operating_system, Ch, type: "LowCardinality(String)" - # field :operating_system_version, Ch, type: "LowCardinality(String)" - # field :browser, Ch, type: "LowCardinality(String)" - # field :browser_version, Ch, type: "LowCardinality(String)" + field :session_screen_size, Ch, type: "LowCardinality(String)", virtual: true + field :session_operating_system, Ch, type: "LowCardinality(String)", virtual: true + field :session_operating_system_version, Ch, type: "LowCardinality(String)", virtual: true + field :session_browser, Ch, type: "LowCardinality(String)", virtual: true + field :session_browser_version, Ch, type: "LowCardinality(String)", virtual: true # field :transferred_from, :string end + @required_keys [:name, :site_id, :hostname, :pathname, :user_id, :timestamp] + @optional_keys [ + :"meta.key", + :"meta.value", + :revenue_source_amount, + :revenue_source_currency, + :revenue_reporting_amount, + :revenue_reporting_currency, + :session_operating_system, + :session_operating_system_version, + :session_browser, + :session_browser_version, + :session_referrer, + :session_referrer_source, + :session_utm_medium, + :session_utm_source, + :session_utm_campaign, + :session_utm_content, + :session_utm_term, + :session_country_code, + :session_subdivision1_code, + :session_subdivision2_code, + :session_city_geoname_id, + :session_screen_size + ] + @all_keys @required_keys ++ @optional_keys + + def update(event, attrs) do + cast(event, attrs, @all_keys) + end + + def validate(event) do + validate_required(event, @required_keys) + end + def new(attrs) do %__MODULE__{} |> cast( diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 98b36e16d..2211c1dff 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -12,9 +12,8 @@ defmodule Plausible.Ingestion.Event do defstruct domain: nil, site: nil, - clickhouse_event_attrs: %{}, clickhouse_session_attrs: %{}, - clickhouse_event: nil, + clickhouse_event: %ClickhouseEventV2{} |> Ecto.Changeset.change(), dropped?: false, drop_reason: nil, request: nil, @@ -32,9 +31,8 @@ defmodule Plausible.Ingestion.Event do @type t() :: %__MODULE__{ domain: String.t() | nil, site: %Plausible.Site{} | nil, - clickhouse_event_attrs: map(), clickhouse_session_attrs: %{}, - clickhouse_event: %ClickhouseEventV2{} | nil, + clickhouse_event: %Ecto.Changeset{}, dropped?: boolean(), drop_reason: drop_reason(), request: Request.t(), @@ -142,7 +140,9 @@ defmodule Plausible.Ingestion.Event do end defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do - struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) + struct!(event, + clickhouse_event: Plausible.ClickhouseEventV2.update(event.clickhouse_event, attrs) + ) end defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do @@ -263,19 +263,16 @@ defmodule Plausible.Ingestion.Event do generate_user_id( event.request, event.domain, - event.clickhouse_event_attrs.hostname, + event.clickhouse_event.changes[:hostname], event.salts.current ) }) end defp validate_clickhouse_event(%__MODULE__{} = event) do - clickhouse_event = - event - |> Map.fetch!(:clickhouse_event_attrs) - |> ClickhouseEventV2.new() + event_asdas = event.clickhouse_event |> Plausible.ClickhouseEventV2.validate() - case Ecto.Changeset.apply_action(clickhouse_event, nil) do + case Ecto.Changeset.apply_action(event_asdas, nil) do {:ok, valid_clickhouse_event} -> %{event | clickhouse_event: valid_clickhouse_event} diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index ca6dba919..8c30236f2 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -205,7 +205,7 @@ defmodule Plausible.Ingestion.EventTest do assert {:ok, request} = Request.build(conn) assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request) - assert event.clickhouse_event_attrs.hostname == "192.168.0.1" + assert event.clickhouse_event.hostname == "192.168.0.1" end test "Hostname is stored with public suffix processing" do @@ -220,6 +220,6 @@ defmodule Plausible.Ingestion.EventTest do assert {:ok, request} = Request.build(conn) assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request) - assert event.clickhouse_event_attrs.hostname == "foo.netlify.app" + assert event.clickhouse_event.hostname == "foo.netlify.app" end end