journey-rebased
Adam Rutkowski 3 years ago
parent 2fcf5a0bb4
commit 61cdfd2529

@ -13,6 +13,7 @@ defmodule Plausible.Ingestion.Event do
defstruct domain: nil,
site: nil,
clickhouse_event_attrs: %{},
clickhouse_session_attrs: %{},
clickhouse_event: nil,
dropped?: false,
drop_reason: nil,
@ -33,6 +34,7 @@ defmodule Plausible.Ingestion.Event do
domain: String.t() | nil,
site: %Plausible.Site{} | nil,
clickhouse_event_attrs: map(),
clickhouse_session_attrs: map(),
clickhouse_event: %ClickhouseEventV2{} | nil,
dropped?: boolean(),
drop_reason: drop_reason(),
@ -161,10 +163,14 @@ defmodule Plausible.Ingestion.Event do
struct!(event, fields)
end
defp update_attrs(%__MODULE__{} = event, %{} = attrs) do
defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do
struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs))
end
defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
end
defp drop_datacenter_ip(%__MODULE__{} = event) do
case event.request.ip_classification do
"dc_ip" ->
@ -197,7 +203,7 @@ defmodule Plausible.Ingestion.Event do
drop(event, :bot)
%UAInspector.Result{} = user_agent ->
update_attrs(event, %{
update_session_attrs(event, %{
operating_system: os_name(user_agent),
operating_system_version: os_version(user_agent),
browser: browser_name(user_agent),
@ -211,7 +217,7 @@ defmodule Plausible.Ingestion.Event do
end
defp put_basic_info(%__MODULE__{} = event) do
update_attrs(event, %{
update_event_attrs(event, %{
domain: event.domain,
site_id: event.site.id,
timestamp: event.request.timestamp,
@ -224,7 +230,7 @@ defmodule Plausible.Ingestion.Event do
defp put_referrer(%__MODULE__{} = event) do
ref = parse_referrer(event.request.uri, event.request.referrer)
update_attrs(event, %{
update_session_attrs(event, %{
referrer_source: get_referrer_source(event.request, ref),
referrer: clean_referrer(ref)
})
@ -233,7 +239,7 @@ defmodule Plausible.Ingestion.Event do
defp put_utm_tags(%__MODULE__{} = event) do
query_params = event.request.query_params
update_attrs(event, %{
update_session_attrs(event, %{
utm_medium: query_params["utm_medium"],
utm_source: query_params["utm_source"],
utm_campaign: query_params["utm_campaign"],
@ -245,17 +251,16 @@ defmodule Plausible.Ingestion.Event do
defp put_geolocation(%__MODULE__{} = event) do
case event.request.ip_classification do
"anonymous_vpn_ip" ->
result = %{country_code: "A1"}
update_attrs(event, result)
update_session_attrs(event, %{country_code: "A1"})
_any ->
result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{}
update_attrs(event, result)
update_session_attrs(event, result)
end
end
defp drop_shield_rule_country(
%__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
)
when is_binary(domain) and is_binary(cc) do
case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
@ -273,7 +278,7 @@ defmodule Plausible.Ingestion.Event do
# defensive: ensuring the keys/values are always in the same order
{keys, values} = Enum.unzip(props)
update_attrs(event, %{
update_event_attrs(event, %{
"meta.key": keys,
"meta.value": values
})
@ -284,7 +289,7 @@ defmodule Plausible.Ingestion.Event do
defp put_revenue(event) do
on_full_build do
attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event)
update_attrs(event, attrs)
update_event_attrs(event, attrs)
else
event
end
@ -295,7 +300,7 @@ defmodule Plausible.Ingestion.Event do
end
defp put_user_id(%__MODULE__{} = event) do
update_attrs(event, %{
update_event_attrs(event, %{
user_id:
generate_user_id(
event.request,

@ -138,6 +138,7 @@ defmodule Plausible.Ingestion.Request do
if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer)
else
changeset

@ -20,9 +20,9 @@ defmodule Plausible.AuthTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Auth.has_active_sites?(user)
end

@ -634,14 +634,25 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
last_30_days: %{
@ -665,14 +676,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
current_cycle: %{
@ -711,14 +737,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
for site <- [site1, site2, site3] do
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
end
assert %{
@ -749,23 +790,29 @@ defmodule Plausible.Billing.QuotaTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:event, timestamp: ~N[2023-04-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-05 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-05-01 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-02 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-03 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-04 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-06-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-05 00:00:00], name: "custom")
])
journey site, now: ~N[2023-04-01 00:00:00] do
custom_event "customx1", idle: &tomorrow/1
custom_event "customx2", idle: &tomorrow/1
custom_event "customx3", idle: &tomorrow/1
custom_event "customx4", idle: &tomorrow/1
custom_event "customx5"
end
journey site, now: ~N[2023-05-01 00:00:00] do
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
end
journey site, now: ~N[2023-06-01 00:00:00] do
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom"
end
{:ok, %{user: user}}
end
@ -807,9 +854,9 @@ defmodule Plausible.Billing.QuotaTest do
]
)
populate_stats(different_site, [
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "custom")
])
journey different_site, now: ~N[2023-05-05 00:00:00] do
custom_event "custom"
end
last_bill_date = ~D[2023-06-03]
today = ~D[2023-06-05]

@ -44,20 +44,15 @@ defmodule Plausible.FunnelsSamplingTest do
)
Plausible.Site.Cache.refresh_all()
journey = MassJourney
journey site, manual: MassJourney, ip: &random_ipv6/0, user_agent: "JourneyBrowser" do
journey site, manual: journey, ip: &random_ipv6/0, user_agent: "JourneyBrowser" do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
1..50_000
|> Task.async_stream(fn _ ->
MassJourney.run()
end)
|> Stream.run()
MassJourney.flush()
journey.run_many(50_000)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})

@ -3,7 +3,6 @@ defmodule Plausible.FunnelsTest do
@moduletag :full_build_only
use Plausible
use Plausible.Test.Support.Journey
on_full_build do
alias Plausible.Goals

@ -95,23 +95,11 @@ defmodule Plausible.PropsTest do
user = insert(:user, subscription: build(:growth_subscription))
site = insert(:site, members: [user])
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
assert {:error, :upgrade_required} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: nil} = Plausible.Repo.reload!(site)
@ -120,23 +108,11 @@ defmodule Plausible.PropsTest do
test "allow_existing_props/1 saves the most frequent prop keys" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, site} = Plausible.Props.allow_existing_props(site)
@ -144,74 +120,39 @@ defmodule Plausible.PropsTest do
Plausible.Repo.reload!(site)
end
test "allow_existing_props/1 skips invalid keys" do
site = insert(:site)
# test "allow_existing_props/1 skips invalid keys" do
# site = insert(:site)
# long = String.duplicate("a", 301)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", String.duplicate("a", 301)],
"meta.value": ["500", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
# journey site do
# custom_event "Payment", props: %{:amount => 500, long => true}
# custom_event "Payment", props: %{amount: 100, logged_in: false}
# custom_event "Payment", props: %{amount: 100, is_customer: false}
# end
{:ok, site} = Plausible.Props.allow_existing_props(site)
# {:ok, site} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
Plausible.Repo.reload!(site)
end
# assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
# Plausible.Repo.reload!(site)
# end
test "allow_existing_props/1 can be run multiple times" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{:amount => 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]}} =
Plausible.Props.allow_existing_props(site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
{:ok,
%Plausible.Site{
@ -229,23 +170,11 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 returns prop keys from events" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
assert ["amount", "first_time_customer", "is_customer", "logged_in", "os", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()
@ -254,23 +183,14 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 does not return internal prop keys from special event types" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "File Download",
"meta.key": ["url", "logged_in"],
"meta.value": ["http://file.test", "false"]
),
build(:event,
name: "Outbound Link: Click",
"meta.key": ["url", "first_time_customer"],
"meta.value": ["http://link.test", "true"]
),
build(:event,
name: "404",
"meta.key": ["path", "with_error"],
"meta.value": ["/i-dont-exist", "true"]
)
])
journey site do
custom_event "File Download", props: %{url: "http://file.test", logged_in: false}
custom_event "Outbound Link: Click",
props: %{url: "http://link.test", first_time_customer: true}
custom_event "404", props: %{path: "/i-dont-exist", with_error: true}
end
assert ["first_time_customer", "logged_in", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()

@ -49,9 +49,9 @@ defmodule Plausible.SitesTest do
test "is date if site does have stats" do
site = insert(:site)
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Sites.stats_start_date(site) == Timex.today(site.timezone)
end

@ -56,14 +56,19 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageviuw "/"
end
assert %{
change: 100,
@ -104,21 +109,31 @@ defmodule Plausible.Stats.ClickhouseTest do
site2 = insert(:site)
site3 = insert(:site)
user_id = 111
journey site1, now: ~N[2023-10-25 13:00:00] do
pageview "/", idle: 60
pageview "/"
end
populate_stats(site1, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:01:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
journey site1, now: ~N[2023-10-25 15:58:00] do
pageview "/"
end
populate_stats(site2, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site1, now: ~N[2023-10-25 15:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageview "/"
end
assert result =
Clickhouse.last_24h_visitors_hourly_intervals([site1, site2, site3], fixed_now)
@ -144,13 +159,25 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-24 11:58:00]),
build(:pageview, timestamp: ~N[2023-10-24 12:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00])
])
journey site, now: ~N[2023-10-24 11:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-24 12:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 50,
@ -164,10 +191,10 @@ defmodule Plausible.Stats.ClickhouseTest do
user_id = 111
populate_stats(site, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 15:59:00] do
pageview "/", idle: 60
pageview "/"
end
result = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]
assert result[:visitors] == 1
@ -177,14 +204,18 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60 * 60
pageview "/"
end
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 100,
@ -225,27 +256,25 @@ defmodule Plausible.Stats.ClickhouseTest do
site = insert(:site)
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
populate_stats(site, [
build(:pageview,
pathname: "/",
referrer_source: "Twitter"
),
build(:pageview,
pathname: "/plausible.io"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Bing"
)
])
journey site do
pageview "/", referrer: "https://twitter.com"
end
journey site do
pageview "/plausible.io"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://bing.com"
end
assert [
%{count: 2, name: "Google"},

@ -3,6 +3,9 @@ defmodule Plausible.Test.Support.Journey do
quote do
require Plausible.Test.Support.Journey
import Plausible.Test.Support.Journey
Module.register_attribute(__MODULE__, :journey_index, accumulate: false)
Module.put_attribute(__MODULE__, :journey_index, 0)
end
end
@ -14,22 +17,33 @@ defmodule Plausible.Test.Support.Journey do
Enum.reduce(journey, state, fn
{:pageview, [url, opts]}, state ->
payload = %{
name: "pageview",
domain: site.domain,
url: build_url(site, url, opts)
}
payload =
%{
name: "pageview",
domain: site.domain,
url: build_url(site, url, opts)
}
|> add_common_body_params(opts)
conn
|> Plug.Adapters.Test.Conn.conn(:post, "/api/events", payload)
|> ingest(state, Keyword.get(opts, :idle, 1))
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
{:custom_event, [name, opts]}, state ->
payload = %{name: name, domain: site.domain, url: build_url(site, "/", opts)}
payload =
%{name: name, domain: site.domain, url: build_url(site, "/", opts)}
|> add_common_body_params(opts)
payload =
if opts[:props] do
Map.put(payload, :props, opts[:props])
else
payload
end
conn
|> conn(:post, "/api/events", payload)
|> ingest(state, Keyword.get(opts, :idle, 1))
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
end)
if !state[:manual] do
@ -37,6 +51,19 @@ defmodule Plausible.Test.Support.Journey do
end
end
defp add_common_body_params(payload, params) do
params =
params
|> Enum.into(%{})
|> Map.take(~w[referrer]a)
if Enum.empty?(params) do
payload
else
Map.merge(payload, params)
end
end
def build_url(site, path, params) do
if params[:url] do
params[:url]
@ -73,14 +100,26 @@ defmodule Plausible.Test.Support.Journey do
defp invoke_if_function(f) when is_function(f, 0), do: f.()
defp invoke_if_function(value), do: value
defp invoke_if_function(f, arg) when is_function(f, 1), do: f.(arg)
defp invoke_if_function(value, _), do: value
def flush_buffers do
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()
end
defp ingest(conn, state, idle_offset) do
defp ingest(conn, state, idle) do
now = invoke_if_function(state.now)
idle_offset =
case idle do
n when is_integer(n) ->
n
%NaiveDateTime{} = ndt ->
NaiveDateTime.diff(ndt, now, :second)
end
{:ok, request} =
Plausible.Ingestion.Request.build(conn, now)
@ -107,26 +146,39 @@ defmodule Plausible.Test.Support.Journey do
end
defmacro journey(site, state \\ [], do: block) do
mod = :"#{__CALLER__.module}.Journey#{:erlang.phash2(binding())}"
__journey__(aliased(state[:manual]) || mod, site, state, block)
idx = Module.get_attribute(__CALLER__.module, :journey_index)
idx = idx + 1
Module.put_attribute(__CALLER__.module, :journey_index, idx)
mod = :"#{__CALLER__.module}.Journey#{idx}"
__journey__(mod, site, state, block, aliased(state[:manual]))
end
defp aliased({:__aliases__, _, mod}), do: Module.concat(mod)
defp aliased(other), do: other
defp __journey__(mod, site, state, block) do
defmacro module_name(mod, site, state, nil) do
:"#{mod}_#{:erlang.phash2({site, state})}"
end
defmacro module_name(_, _, _, manual) do
manual
end
defp __journey__(mod, site, state, block, mod_override) do
quote do
defmodule unquote(mod) do
defmodule :"#{module_name(unquote(mod), unquote(site), unquote(state), unquote(mod_override))}" do
Module.register_attribute(__MODULE__, :journey, accumulate: true)
@site unquote(site)
@initial_state default(Enum.into(unquote(state), %{}))
unquote(block)
def run(overrides \\ %{}) do
def run() do
Plausible.Test.Support.Journey.run(
@site,
Map.merge(@initial_state, overrides),
@initial_state,
Enum.reverse(@journey)
)
end
@ -134,10 +186,18 @@ defmodule Plausible.Test.Support.Journey do
def flush() do
Plausible.Test.Support.Journey.flush_buffers()
end
def run_many(n) do
1..n
|> Task.async_stream(fn _ -> run() end)
|> Stream.run()
flush()
end
end
if is_nil(unquote(state)[:manual]) do
unquote(mod).run()
:"#{module_name(unquote(mod), unquote(site), unquote(state), unquote(mod_override))}".run()
end
end
end

@ -6,6 +6,7 @@ defmodule Plausible.TestUtils do
quote do
require Plausible.TestUtils
import Plausible.TestUtils
use Plausible.Test.Support.Journey
end
end
@ -265,4 +266,8 @@ defmodule Plausible.TestUtils do
[a1, a2, a3, a4, a5, a6, a7, a8]
|> Enum.map_join(":", &Base.encode16(<<&1>>, case: :lower))
end
def tomorrow(now) do
NaiveDateTime.add(now, 1, :day)
end
end

@ -77,15 +77,14 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, domain: "test-site.com", inserted_at: Timex.shift(now, days: -8))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview,
user_id: 123,
timestamp: Timex.shift(now, days: -7),
referrer_source: "Google"
),
build(:pageview, user_id: 123, timestamp: Timex.shift(now, days: -7)),
build(:pageview, timestamp: Timex.shift(now, days: -7))
])
journey site, now: Timex.shift(now, days: -7) do
pageview "/", referrer: "https://google.com"
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -110,12 +109,19 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, user_id: 1, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -145,12 +151,18 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, user_id: 1, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: two_weeks_ago do
pageview "/"
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -180,10 +192,13 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})

Loading…
Cancel
Save