Compare commits

...

19 Commits

Author SHA1 Message Date
Adam Rutkowski 1af9d02829 Add debug mode 3 years ago
Adam Rutkowski 3144ebb932 wip 3 years ago
Adam Rutkowski 106699b936 wip 3 years ago
Adam Rutkowski b88bbaac1c wip 3 years ago
Adam Rutkowski 5411b0d8e5 wip 3 years ago
Adam Rutkowski 0525362992 wip 3 years ago
Adam Rutkowski 26839305f6 wip 3 years ago
Adam Rutkowski 4a37cf6401 pass 3 years ago
Adam Rutkowski 7360f98624 wip 3 years ago
Adam Rutkowski 185a90c9c2 wip 3 years ago
Adam Rutkowski e0b285f8c3 wip 3 years ago
Adam Rutkowski b1e63fa2f8 wip 3 years ago
Adam Rutkowski cb64a05f8b wip 3 years ago
Adam Rutkowski f19733b4dd wip 3 years ago
Adam Rutkowski badfeb0ad5 wip 3 years ago
Adam Rutkowski 875ddf68e8 satanic 3 years ago
Adam Rutkowski 60b9ca7d16 fixup 3 years ago
Adam Rutkowski 48e795ce98 wip 3 years ago
Adam Rutkowski dba8a02eee Proof of concept 3 years ago

@ -2,5 +2,11 @@
plugins: [Phoenix.LiveView.HTMLFormatter],
import_deps: [:ecto, :ecto_sql, :phoenix],
subdirectories: ["priv/*/migrations"],
inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"]
inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
locals_without_parens: [
pageview: 1,
pageview: 2,
custom_event: 1,
custom_event: 2
]
]

@ -138,6 +138,7 @@ defmodule Plausible.Ingestion.Request do
if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer)
else
changeset

@ -38,6 +38,8 @@ defmodule Plausible.Shield.IPRuleCache do
end
@impl true
def get_from_source({_, "0.0.0.0"}), do: nil
def get_from_source({domain, address}) do
query =
base_db_query()

@ -20,9 +20,9 @@ defmodule Plausible.AuthTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Auth.has_active_sites?(user)
end

@ -634,14 +634,25 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
last_30_days: %{
@ -665,14 +676,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
current_cycle: %{
@ -711,14 +737,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
for site <- [site1, site2, site3] do
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
end
assert %{
@ -749,23 +790,29 @@ defmodule Plausible.Billing.QuotaTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:event, timestamp: ~N[2023-04-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-05 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-05-01 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-02 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-03 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-04 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-06-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-05 00:00:00], name: "custom")
])
journey site, now: ~N[2023-04-01 00:00:00] do
custom_event "customx1", idle: &tomorrow/1
custom_event "customx2", idle: &tomorrow/1
custom_event "customx3", idle: &tomorrow/1
custom_event "customx4", idle: &tomorrow/1
custom_event "customx5"
end
journey site, now: ~N[2023-05-01 00:00:00] do
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
end
journey site, now: ~N[2023-06-01 00:00:00] do
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom"
end
{:ok, %{user: user}}
end
@ -807,9 +854,9 @@ defmodule Plausible.Billing.QuotaTest do
]
)
populate_stats(different_site, [
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "custom")
])
journey different_site, now: ~N[2023-05-05 00:00:00] do
custom_event "custom"
end
last_bill_date = ~D[2023-06-03]
today = ~D[2023-06-05]

@ -0,0 +1,63 @@
defmodule Plausible.FunnelsSamplingTest do
use Plausible.DataCase, async: false
@moduletag :full_build_only
use Plausible
use Plausible.Test.Support.Journey
# This test does a massive ingest, so we make sure the caches are utilized
# (both Sites.Cache and UA parsing). We also ingest on all cores, where only
# initially SipHash is a slow-down but warms up later on.
# On 24 cores it takes around 5.5s.
on_full_build do
alias Plausible.Goals
alias Plausible.Funnels
alias Plausible.Stats
setup do
site = insert(:site)
{:ok, g1} = Goals.create(site, %{"page_path" => "/go/to/blog/**"})
{:ok, g2} = Goals.create(site, %{"event_name" => "Signup"})
{:ok, g3} = Goals.create(site, %{"page_path" => "/checkout"})
{:ok,
%{
site: site,
goals: [g1, g2, g3],
steps: [g1, g2, g3] |> Enum.map(&%{"goal_id" => &1.id})
}}
end
setup_patch_env(Plausible.Cache, enabled: true)
test "sampling", %{steps: [g1, g2, g3]} do
site = insert(:site, domain: "1.1.1.1")
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup",
[g1, g2, g3]
)
Plausible.Site.Cache.refresh_all()
journey = MassJourney
journey site, manual: journey, ip: &random_ipv6/0, user_agent: "JourneyBrowser" do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey.run_many(50_000)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
{:ok, funnel_data} = Stats.funnel(site, query, funnel.id)
assert_in_delta funnel_data[:all_visitors], 50_000, 5000
end
end
end

@ -1,5 +1,5 @@
defmodule Plausible.FunnelsTest do
use Plausible.DataCase
use Plausible.DataCase, async: true
@moduletag :full_build_only
use Plausible
@ -171,14 +171,20 @@ defmodule Plausible.FunnelsTest do
]
)
populate_stats(site, [
build(:pageview, pathname: "/irrelevant/page/not/in/funnel", user_id: 999),
build(:pageview, pathname: "/go/to/blog/foo", user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, pathname: "/checkout", user_id: 123),
build(:pageview, pathname: "/go/to/blog/bar", user_id: 666),
build(:event, name: "Signup", user_id: 666)
])
journey site do
pageview "/irrelevant/page/not/in/funnel"
end
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey site do
pageview "/go/to/blog/bar"
custom_event "Signup"
end
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
@ -231,13 +237,16 @@ defmodule Plausible.FunnelsTest do
[g1, g2, g3]
)
populate_stats(site, [
build(:pageview, pathname: "/go/to/blog/foo", user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, pathname: "/checkout", user_id: 123),
build(:pageview, pathname: "/go/to/blog/bar", user_id: 666),
build(:event, name: "Signup", user_id: 666)
])
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
end
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
@ -328,36 +337,6 @@ defmodule Plausible.FunnelsTest do
]
}} = funnel_data
end
@tag :slow
test "sampling", %{site: site, steps: [g1, g2, g3 | _]} do
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup and purchase",
[g1, g2, g3]
)
stats =
1..50_000
|> Enum.flat_map(fn _ ->
user_id = SipHash.hash!("0123456789ABCDEF", :crypto.strong_rand_bytes(64))
[
build(:pageview, pathname: "/go/to/blog/foo", user_id: user_id),
build(:event, name: "Signup", user_id: user_id),
build(:pageview, pathname: "/checkout", user_id: user_id)
]
end)
populate_stats(site, stats)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
{:ok, funnel_data} = Stats.funnel(site, query, funnel.id)
assert_in_delta funnel_data[:all_visitors], 50_000, 5000
end
end
end
end

@ -95,23 +95,11 @@ defmodule Plausible.PropsTest do
user = insert(:user, subscription: build(:growth_subscription))
site = insert(:site, members: [user])
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
assert {:error, :upgrade_required} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: nil} = Plausible.Repo.reload!(site)
@ -120,23 +108,11 @@ defmodule Plausible.PropsTest do
test "allow_existing_props/1 saves the most frequent prop keys" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, site} = Plausible.Props.allow_existing_props(site)
@ -144,74 +120,39 @@ defmodule Plausible.PropsTest do
Plausible.Repo.reload!(site)
end
test "allow_existing_props/1 skips invalid keys" do
site = insert(:site)
# test "allow_existing_props/1 skips invalid keys" do
# site = insert(:site)
# long = String.duplicate("a", 301)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", String.duplicate("a", 301)],
"meta.value": ["500", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
# journey site do
# custom_event "Payment", props: %{:amount => 500, long => true}
# custom_event "Payment", props: %{amount: 100, logged_in: false}
# custom_event "Payment", props: %{amount: 100, is_customer: false}
# end
{:ok, site} = Plausible.Props.allow_existing_props(site)
# {:ok, site} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
Plausible.Repo.reload!(site)
end
# assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
# Plausible.Repo.reload!(site)
# end
test "allow_existing_props/1 can be run multiple times" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{:amount => 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]}} =
Plausible.Props.allow_existing_props(site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
{:ok,
%Plausible.Site{
@ -229,23 +170,11 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 returns prop keys from events" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
assert ["amount", "first_time_customer", "is_customer", "logged_in", "os", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()
@ -254,23 +183,14 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 does not return internal prop keys from special event types" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "File Download",
"meta.key": ["url", "logged_in"],
"meta.value": ["http://file.test", "false"]
),
build(:event,
name: "Outbound Link: Click",
"meta.key": ["url", "first_time_customer"],
"meta.value": ["http://link.test", "true"]
),
build(:event,
name: "404",
"meta.key": ["path", "with_error"],
"meta.value": ["/i-dont-exist", "true"]
)
])
journey site do
custom_event "File Download", props: %{url: "http://file.test", logged_in: false}
custom_event "Outbound Link: Click",
props: %{url: "http://link.test", first_time_customer: true}
custom_event "404", props: %{path: "/i-dont-exist", with_error: true}
end
assert ["first_time_customer", "logged_in", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()

@ -49,9 +49,9 @@ defmodule Plausible.SitesTest do
test "is date if site does have stats" do
site = insert(:site)
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Sites.stats_start_date(site) == Timex.today(site.timezone)
end

@ -56,14 +56,18 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageview "/"
end
assert %{
change: 100,
@ -104,21 +108,31 @@ defmodule Plausible.Stats.ClickhouseTest do
site2 = insert(:site)
site3 = insert(:site)
user_id = 111
journey site1, now: ~N[2023-10-25 13:00:00] do
pageview "/", idle: 60
pageview "/"
end
populate_stats(site1, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:01:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
journey site1, now: ~N[2023-10-25 15:58:00] do
pageview "/"
end
populate_stats(site2, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site1, now: ~N[2023-10-25 15:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageview "/"
end
assert result =
Clickhouse.last_24h_visitors_hourly_intervals([site1, site2, site3], fixed_now)
@ -144,13 +158,25 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-24 11:58:00]),
build(:pageview, timestamp: ~N[2023-10-24 12:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00])
])
journey site, now: ~N[2023-10-24 11:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-24 12:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 50,
@ -164,10 +190,10 @@ defmodule Plausible.Stats.ClickhouseTest do
user_id = 111
populate_stats(site, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 15:59:00] do
pageview "/", idle: 60
pageview "/"
end
result = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]
assert result[:visitors] == 1
@ -177,14 +203,18 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60 * 60
pageview "/"
end
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 100,
@ -225,27 +255,25 @@ defmodule Plausible.Stats.ClickhouseTest do
site = insert(:site)
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
populate_stats(site, [
build(:pageview,
pathname: "/",
session_referrer_source: "Twitter"
),
build(:pageview,
pathname: "/plausible.io"
),
build(:pageview,
pathname: "/plausible.io",
session_referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
session_referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
session_referrer_source: "Bing"
)
])
journey site do
pageview "/", referrer: "https://twitter.com"
end
journey site do
pageview "/plausible.io"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://bing.com"
end
assert [
%{count: 2, name: "Google"},

@ -0,0 +1,229 @@
defmodule Plausible.Test.Support.Journey do
@moduledoc "TODO"
defmacro __using__(_) do
quote do
require Plausible.Test.Support.Journey
import Plausible.Test.Support.Journey
Module.register_attribute(__MODULE__, :journey_index, accumulate: false)
Module.put_attribute(__MODULE__, :journey_index, 0)
end
end
import Plug.Adapters.Test.Conn, only: [conn: 4]
import Plug.Conn
def run(site, state, journey) do
conn = new_conn(state)
# credo:disable-for-this-file Credo.Check.Warning.UnusedEnumOperation
Enum.reduce(journey, state, fn
{:pageview, [url, opts]}, state ->
payload =
%{
name: "pageview",
domain: site.domain,
url: build_url(site, url, opts)
}
|> add_common_body_params(opts)
conn
|> Plug.Adapters.Test.Conn.conn(:post, "/api/events", payload)
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
{:custom_event, [name, opts]}, state ->
payload =
%{name: name, domain: site.domain, url: build_url(site, "/", opts)}
|> add_common_body_params(opts)
payload =
if opts[:props] do
Map.put(payload, :props, opts[:props])
else
payload
end
conn
|> conn(:post, "/api/events", payload)
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
end)
if !state[:manual] do
flush_buffers()
end
end
defp add_common_body_params(payload, params) do
params =
params
|> Enum.into(%{})
|> Map.take(~w[referrer]a)
if Enum.empty?(params) do
payload
else
Map.merge(payload, params)
end
end
def build_url(site, path, params) do
if params[:url] do
params[:url]
else
site_domain = site.domain |> URI.encode_www_form()
query_string =
params
|> Enum.into(%{})
|> Map.take(~w[utm_source utm_medium utm_campaign utm_term utm_content]a)
|> URI.encode_query()
uri = URI.new!("https://" <> Path.join(site_domain, path))
uri = %{uri | query: if(query_string != "", do: query_string)}
to_string(uri)
end
end
defp new_conn(state) do
(state.conn || %Plug.Conn{})
|> put_req_header("content-type", "application/json")
|> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip))
|> maybe_add_header("user-agent", invoke_if_function(state.user_agent))
end
defp maybe_add_header(conn, _header, nil) do
conn
end
defp maybe_add_header(conn, header, value) do
put_req_header(conn, header, value)
end
defp invoke_if_function(f) when is_function(f, 0), do: f.()
defp invoke_if_function(value), do: value
defp invoke_if_function(f, arg) when is_function(f, 1), do: f.(arg)
defp invoke_if_function(value, _), do: value
def flush_buffers do
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()
end
defp ingest(conn, state, idle) do
now = invoke_if_function(state.now)
idle_offset =
case idle do
n when is_integer(n) ->
n
%NaiveDateTime{} = ndt ->
NaiveDateTime.diff(ndt, now, :second)
end
{:ok, request} =
Plausible.Ingestion.Request.build(conn, now)
Plausible.Ingestion.Event.build_and_buffer(request)
new_now = NaiveDateTime.add(now, idle_offset, :second)
if state.debug do
debug(conn, state, idle, now, new_now)
end
Map.put(state, :now, new_now)
end
defp debug(conn, state, idle, now, new_now) do
IO.puts("\n[#{now}] Request:" <> IO.ANSI.yellow())
IO.puts(" #{conn.method} #{conn.request_path}?#{conn.query_string}" <> IO.ANSI.cyan())
IO.puts(" user-agent: #{state.user_agent}")
IO.puts(" x-forwarded-for: #{state.ip}" <> IO.ANSI.green())
IO.puts(" #{Jason.encode!(conn.body_params)}" <> IO.ANSI.reset())
IO.puts("[#{new_now}] Idle complete (#{inspect(idle)})")
end
defmacro default(state) do
h1 = :erlang.phash2(__CALLER__.module, 256)
h2 = :erlang.phash2(__CALLER__.line, 256)
default_ip = "#{h1}.#{h2}.#{h1}.#{h2}"
default_user_agent = "JourneyBrowser #{__CALLER__.module}/#{__CALLER__.line}"
quote do
unquote(state)
|> Map.update(:now, &NaiveDateTime.utc_now/0, & &1)
|> Map.update(:conn, nil, & &1)
|> Map.update(:ip, unquote(default_ip), & &1)
|> Map.update(:user_agent, unquote(default_user_agent), & &1)
|> Map.update(:debug, false, & &1)
end
end
defmacro journey(site, state \\ [], do: block) do
Code.compiler_options(ignore_module_conflict: true)
idx = Module.get_attribute(__CALLER__.module, :journey_index)
idx = idx + 1
Module.put_attribute(__CALLER__.module, :journey_index, idx)
mod = :"#{__CALLER__.module}.Journey#{idx}"
__journey__(aliased(state[:manual]) || mod, site, state, block)
end
defp aliased({:__aliases__, _, mod}), do: Module.concat(mod)
defp aliased(other), do: other
defp __journey__(mod, site, state, block) do
quote do
defmodule unquote(mod) do
Module.register_attribute(__MODULE__, :journey, accumulate: true)
@site unquote(site)
@initial_state default(Enum.into(unquote(state), %{}))
unquote(block)
def run() do
Plausible.Test.Support.Journey.run(
@site,
@initial_state,
Enum.reverse(@journey)
)
end
def flush() do
Plausible.Test.Support.Journey.flush_buffers()
end
def run_many(n) do
1..n
|> Task.async_stream(fn _ -> run() end)
|> Stream.run()
flush()
end
end
if is_nil(unquote(state)[:manual]) do
unquote(mod).run()
end
end
end
defmacro pageview(url, opts \\ []) do
quote do: store(:pageview, [unquote(url), unquote(opts)])
end
defmacro custom_event(name, opts \\ []) do
quote do: store(:custom_event, [unquote(name), unquote(opts)])
end
defmacro store(op, args) do
quote do
Module.put_attribute(__MODULE__, :journey, {unquote(op), unquote(args)})
end
end
end

@ -6,6 +6,7 @@ defmodule Plausible.TestUtils do
quote do
require Plausible.TestUtils
import Plausible.TestUtils
use Plausible.Test.Support.Journey
end
end
@ -280,4 +281,16 @@ defmodule Plausible.TestUtils do
def random_ip() do
Enum.map_join(1..4, ".", fn _ -> Enum.random(1..254) end)
end
def random_ipv6() do
<<a1::16, a2::16, a3::16, a4::16, a5::16, a6::16, a7::16, a8::16>> =
:crypto.strong_rand_bytes(16)
[a1, a2, a3, a4, a5, a6, a7, a8]
|> Enum.map_join(":", &Base.encode16(<<&1>>, case: :lower))
end
def tomorrow(now) do
NaiveDateTime.add(now, 1, :day)
end
end

@ -77,15 +77,14 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, domain: "test-site.com", inserted_at: Timex.shift(now, days: -8))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview,
user_id: 123,
timestamp: Timex.shift(now, days: -7),
session_referrer_source: "Google"
),
build(:pageview, user_id: 123, timestamp: Timex.shift(now, days: -7)),
build(:pageview, timestamp: Timex.shift(now, days: -7))
])
journey site, now: Timex.shift(now, days: -7) do
pageview "/", referrer: "https://google.com"
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -110,12 +109,18 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, user_id: 1, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -145,12 +150,18 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, user_id: 1, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: two_weeks_ago do
pageview "/"
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -180,10 +191,13 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})

Loading…
Cancel
Save