Write event table session columns (#3865)

* Write event table session columns

* Update testing factory rig
pull/3867/head r2024.03.06-383
Karl-Aksel Puulmann 3 years ago committed by GitHub
parent 70d21f1b69
commit c60a2faee4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -23,26 +23,25 @@ defmodule Plausible.ClickhouseEventV2 do
field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_reporting_currency, Ch, type: "FixedString(3)"
# Fields which are in the schema but not managed by us anymore.
# field :referrer, :string
# field :referrer_source, :string
# field :utm_medium, :string
# field :utm_source, :string
# field :utm_campaign, :string
# field :utm_content, :string
# field :utm_term, :string
# Session attributes
field :referrer, :string
field :referrer_source, :string
field :utm_medium, :string
field :utm_source, :string
field :utm_campaign, :string
field :utm_content, :string
field :utm_term, :string
# field :country_code, Ch, type: "FixedString(2)"
# field :subdivision1_code, Ch, type: "LowCardinality(String)"
# field :subdivision2_code, Ch, type: "LowCardinality(String)"
# field :city_geoname_id, Ch, type: "UInt32"
field :country_code, Ch, type: "FixedString(2)"
field :subdivision1_code, Ch, type: "LowCardinality(String)"
field :subdivision2_code, Ch, type: "LowCardinality(String)"
field :city_geoname_id, Ch, type: "UInt32"
# field :screen_size, Ch, type: "LowCardinality(String)"
# field :operating_system, Ch, type: "LowCardinality(String)"
# field :operating_system_version, Ch, type: "LowCardinality(String)"
# field :browser, Ch, type: "LowCardinality(String)"
# field :browser_version, Ch, type: "LowCardinality(String)"
# field :transferred_from, :string
field :screen_size, Ch, type: "LowCardinality(String)"
field :operating_system, Ch, type: "LowCardinality(String)"
field :operating_system_version, Ch, type: "LowCardinality(String)"
field :browser, Ch, type: "LowCardinality(String)"
field :browser_version, Ch, type: "LowCardinality(String)"
end
def new(attrs) do

@ -43,6 +43,26 @@ defmodule Plausible.Ingestion.Event do
changeset: %Ecto.Changeset{}
}
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do
processed_events =
@ -315,15 +335,18 @@ defmodule Plausible.Ingestion.Event do
event.salts.previous
)
session_id =
session =
Plausible.Session.CacheStore.on_event(
event.clickhouse_event,
event.clickhouse_session_attrs,
previous_user_id
)
clickhouse_event = Map.put(event.clickhouse_event, :session_id, session_id)
%{event | clickhouse_event: clickhouse_event}
%{
event
| clickhouse_event:
Map.merge(event.clickhouse_event, Map.take(session, @session_properties))
}
end
defp write_to_buffer(%__MODULE__{clickhouse_event: clickhouse_event} = event) do

@ -5,18 +5,15 @@ defmodule Plausible.Session.CacheStore do
def on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) do
found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)
session =
if found_session do
updated_session = update_session(found_session, event)
buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])
persist_session(updated_session)
else
new_session = new_session_from_event(event, session_attributes)
buffer.insert([new_session])
persist_session(new_session)
end
session.session_id
if found_session do
updated_session = update_session(found_session, event)
buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}])
persist_session(updated_session)
else
new_session = new_session_from_event(event, session_attributes)
buffer.insert([new_session])
persist_session(new_session)
end
end
defp find_session(_domain, nil), do: nil

@ -226,7 +226,7 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
assert event.clickhouse_event_attrs.hostname == "192.168.0.1"
assert event.clickhouse_event.hostname == "192.168.0.1"
end
test "Hostname is stored with public suffix processing" do
@ -241,6 +241,6 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [event]}} = Event.build_and_buffer(request)
assert event.clickhouse_event_attrs.hostname == "foo.netlify.app"
assert event.clickhouse_event.hostname == "foo.netlify.app"
end
end

@ -12,7 +12,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site}
end
test "records the event", %{conn: conn, site: site} do
test "records the event and session", %{conn: conn, site: site} do
params = %{
domain: site.domain,
name: "pageview",
@ -26,6 +26,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
pageview = get_event(site)
session = get_created_session(site)
assert response(conn, 202) == "ok"
assert pageview.hostname == "example.com"
@ -33,6 +34,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.site_id == site.id
assert pageview.pathname == "/"
assert pageview.session_id == session.session_id
end
test "works with Content-Type: text/plain", %{conn: conn, site: site} do
@ -224,12 +226,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
event = get_event(site)
assert response(conn, 202) == "ok"
assert session.operating_system == "Mac"
assert session.operating_system_version == "10.13"
assert session.browser == "Chrome"
assert session.browser_version == "70.0"
assert event.operating_system == "Mac"
assert event.operating_system_version == "10.13"
assert event.browser == "Chrome"
assert event.browser_version == "70.0"
end
test "parses referrer", %{conn: conn, site: site} do
@ -265,10 +272,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
event = get_event(site)
assert response(conn, 202) == "ok"
assert session.referrer == "facebook.com/page"
assert session.referrer_source == "Facebook"
assert event.referrer == session.referrer
assert event.referrer_source == session.referrer_source
end
test "ignores event when referrer is a spammer", %{conn: conn, site: site} do
@ -811,11 +821,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
session = get_created_session(site)
event = get_event(site)
assert session.country_code == "GB"
assert session.subdivision1_code == "GB-ENG"
assert session.subdivision2_code == "GB-WBK"
assert session.city_geoname_id == 2_655_045
assert event.country_code == session.country_code
assert event.subdivision1_code == session.subdivision1_code
assert event.subdivision2_code == session.subdivision2_code
assert event.city_geoname_id == session.city_geoname_id
end
test "ignores unknown country code ZZ", %{conn: conn, site: site} do
@ -1108,6 +1123,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.hostname == "test.com"
assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ"
assert session.utm_source == "%balle%"
assert pageview.utm_source == session.utm_source
end
test "can use double quotes in query params", %{conn: conn, site: site} do

@ -89,14 +89,13 @@ defmodule Plausible.Factory do
def event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
%{
%Plausible.ClickhouseEventV2{
hostname: hostname,
site_id: Enum.random(1000..10_000),
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
_factory_event: true
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
end

@ -82,8 +82,9 @@ defmodule Plausible.TestUtils do
|> Map.put(:site_id, pageview.site.id)
Factory.build(:pageview, pageview)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
@ -93,8 +94,9 @@ defmodule Plausible.TestUtils do
events =
Enum.map(events, fn event ->
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
@ -148,7 +150,7 @@ defmodule Plausible.TestUtils do
event = Map.put(event, :site_id, site.id)
case event do
%{_factory_event: true} ->
%Plausible.ClickhouseEventV2{} ->
event
imported_event ->
@ -179,7 +181,7 @@ defmodule Plausible.TestUtils do
end)
|> Enum.split_with(fn event ->
case event do
%{_factory_event: true} ->
%Plausible.ClickhouseEventV2{} ->
true
_ ->
@ -192,20 +194,13 @@ defmodule Plausible.TestUtils do
end
defp populate_native_stats(events) do
sessions =
Enum.reduce(events, %{}, fn event, sessions ->
session_id = Plausible.Session.CacheStore.on_event(event, event, nil)
Map.put(sessions, {event.site_id, event.user_id}, session_id)
end)
Enum.each(events, fn event ->
clickhouse_event =
%Plausible.ClickhouseEventV2{}
|> Map.merge(event)
|> Map.put(:session_id, sessions[{event.site_id, event.user_id}])
for event_params <- events do
session = Plausible.Session.CacheStore.on_event(event_params, event_params, nil)
Plausible.Event.WriteBuffer.insert(clickhouse_event)
end)
event_params
|> Map.merge(session)
|> Plausible.Event.WriteBuffer.insert()
end
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()

Loading…
Cancel
Save