Remove old columns from being accessible in elixir code

drop-event-columns
Karl-Aksel Puulmann 3 years ago
parent dc84a32aa4
commit 7fcb6a8c9b

@ -15,25 +15,6 @@ defmodule Plausible.ClickhouseEventV2 do
field :session_id, Ch, type: "UInt64"
field :timestamp, :naive_datetime
field :referrer, :string
field :referrer_source, :string
field :utm_medium, :string
field :utm_source, :string
field :utm_campaign, :string
field :utm_content, :string
field :utm_term, :string
field :country_code, Ch, type: "FixedString(2)"
field :subdivision1_code, Ch, type: "LowCardinality(String)"
field :subdivision2_code, Ch, type: "LowCardinality(String)"
field :city_geoname_id, Ch, type: "UInt32"
field :screen_size, Ch, type: "LowCardinality(String)"
field :operating_system, Ch, type: "LowCardinality(String)"
field :operating_system_version, Ch, type: "LowCardinality(String)"
field :browser, Ch, type: "LowCardinality(String)"
field :browser_version, Ch, type: "LowCardinality(String)"
field :"meta.key", {:array, :string}
field :"meta.value", {:array, :string}
@ -42,7 +23,26 @@ defmodule Plausible.ClickhouseEventV2 do
field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_reporting_currency, Ch, type: "FixedString(3)"
field :transferred_from, :string
# Fields which are in the schema but not managed by us anymore.
# field :referrer, :string
# field :referrer_source, :string
# field :utm_medium, :string
# field :utm_source, :string
# field :utm_campaign, :string
# field :utm_content, :string
# field :utm_term, :string
# field :country_code, Ch, type: "FixedString(2)"
# field :subdivision1_code, Ch, type: "LowCardinality(String)"
# field :subdivision2_code, Ch, type: "LowCardinality(String)"
# field :city_geoname_id, Ch, type: "UInt32"
# field :screen_size, Ch, type: "LowCardinality(String)"
# field :operating_system, Ch, type: "LowCardinality(String)"
# field :operating_system_version, Ch, type: "LowCardinality(String)"
# field :browser, Ch, type: "LowCardinality(String)"
# field :browser_version, Ch, type: "LowCardinality(String)"
# field :transferred_from, :string
end
def new(attrs) do
@ -56,22 +56,6 @@ defmodule Plausible.ClickhouseEventV2 do
:pathname,
:user_id,
:timestamp,
:operating_system,
:operating_system_version,
:browser,
:browser_version,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:"meta.key",
:"meta.value",
:revenue_source_amount,

@ -69,24 +69,20 @@ defmodule Plausible.Factory do
end
def pageview_factory do
struct!(
event_factory(),
%{
name: "pageview"
}
)
Map.put(event_factory(), :name, "pageview")
end
def event_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
%Plausible.ClickhouseEventV2{
%{
hostname: hostname,
site_id: Enum.random(1000..10_000),
pathname: "/",
timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
_factory_event: true
}
end

@ -82,9 +82,8 @@ defmodule Plausible.TestUtils do
|> Map.put(:site_id, pageview.site.id)
Factory.build(:pageview, pageview)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
@ -94,9 +93,8 @@ defmodule Plausible.TestUtils do
events =
Enum.map(events, fn event ->
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> Map.delete(:_factory_event)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
@ -150,7 +148,7 @@ defmodule Plausible.TestUtils do
event = Map.put(event, :site_id, site.id)
case event do
%Plausible.ClickhouseEventV2{} ->
%{_factory_event: true} ->
event
imported_event ->
@ -181,7 +179,7 @@ defmodule Plausible.TestUtils do
end)
|> Enum.split_with(fn event ->
case event do
%Plausible.ClickhouseEventV2{} ->
%{_factory_event: true} ->
true
_ ->
@ -196,14 +194,42 @@ defmodule Plausible.TestUtils do
defp populate_native_stats(events) do
sessions =
Enum.reduce(events, %{}, fn event, sessions ->
# :TODO: Handle session parameters separately
session_id = Plausible.Session.CacheStore.on_event(event, event, nil)
# :KLUDGE: Callsites should not use :referrer, etc directly and instead
# nest it under session: %{ referrer: "foo" }. This is left here
# for backwards compatibility / to avoid breaking too many PRs.
session_params =
event
|> Map.take([
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
])
|> Map.merge(Map.get(event, :session, %{}))
session_id = Plausible.Session.CacheStore.on_event(event, session_params, nil)
Map.put(sessions, {event.site_id, event.user_id}, session_id)
end)
Enum.each(events, fn event ->
event = Map.put(event, :session_id, sessions[{event.site_id, event.user_id}])
Plausible.Event.WriteBuffer.insert(event)
clickhouse_event =
%Plausible.ClickhouseEventV2{}
|> Map.merge(event)
|> Map.put(:session_id, sessions[{event.site_id, event.user_id}])
Plausible.Event.WriteBuffer.insert(clickhouse_event)
end)
Plausible.Session.WriteBuffer.flush()

Loading…
Cancel
Save