Compare commits

...

22 Commits

Author SHA1 Message Date
Adam Rutkowski 9676d3ebd1 PoC - device preparation 3 years ago
Adam Rutkowski 7e8fa24f8f wip 3 years ago
Adam Rutkowski 641e3f636c wip 3 years ago
Adam Rutkowski 5b228d65bc Add debug mode 3 years ago
Adam Rutkowski 4e1ffba5f5 wip 3 years ago
Adam Rutkowski b8f2c14332 wip 3 years ago
Adam Rutkowski 1f1a6d96b7 wip 3 years ago
Adam Rutkowski 0aada2632e wip 3 years ago
Adam Rutkowski 79d427a4a6 wip 3 years ago
Adam Rutkowski 39aef3447b wip 3 years ago
Adam Rutkowski dd19966600 pass 3 years ago
Adam Rutkowski 04d01cc860 wip 3 years ago
Adam Rutkowski b5d6c917a9 wip 3 years ago
Adam Rutkowski d60a18dc12 wip 3 years ago
Adam Rutkowski 00b9213d0c wip 3 years ago
Adam Rutkowski 61cdfd2529 wip 3 years ago
Adam Rutkowski 2fcf5a0bb4 wip 3 years ago
Adam Rutkowski 70063bc79e wip 3 years ago
Adam Rutkowski 1bf2be88d3 satanic 3 years ago
Adam Rutkowski be6aa3ef4c fixup 3 years ago
Adam Rutkowski 1033334279 wip 3 years ago
Adam Rutkowski 0f95999eb1 Proof of concept 3 years ago

@ -2,5 +2,11 @@
plugins: [Phoenix.LiveView.HTMLFormatter],
import_deps: [:ecto, :ecto_sql, :phoenix],
subdirectories: ["priv/*/migrations"],
inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"]
inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
locals_without_parens: [
pageview: 1,
pageview: 2,
custom_event: 1,
custom_event: 2
]
]

@ -160,6 +160,7 @@ defmodule Plausible.Ingestion.Event do
|> Keyword.put(:drop_reason, reason)
emit_telemetry_dropped(event, reason)
IO.inspect(reason, label: :dropped)
struct!(event, fields)
end

@ -138,6 +138,7 @@ defmodule Plausible.Ingestion.Request do
if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer)
else
changeset

@ -38,6 +38,8 @@ defmodule Plausible.Shield.IPRuleCache do
end
@impl true
def get_from_source({_, "0.0.0.0"}), do: nil
def get_from_source({domain, address}) do
query =
base_db_query()

@ -7,7 +7,7 @@ defmodule Plausible.Timezones do
end
@spec to_utc_datetime(NaiveDateTime.t(), String.t()) :: DateTime.t()
def to_utc_datetime(naive_date_time, timezone) do
def to_utc_datetime(naive_date_time, timezone \\ "UTC") do
case Timex.to_datetime(naive_date_time, timezone) do
%DateTime{} = tz_dt ->
Timex.Timezone.convert(tz_dt, "UTC")

@ -149,6 +149,12 @@ defmodule Plausible.MixProject do
setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"],
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"],
"seed.journey": [
"ecto.drop",
"ecto.create",
"ecto.migrate",
"run priv/repo/seeds_journey.exs"
],
test: ["ecto.create --quiet", "ecto.migrate", "test", "clean_clickhouse"],
sentry_recompile: ["compile", "deps.compile sentry --force"],
"assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"],

@ -20,9 +20,9 @@ defmodule Plausible.AuthTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Auth.has_active_sites?(user)
end

@ -634,14 +634,25 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
last_30_days: %{
@ -665,14 +676,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
assert %{
current_cycle: %{
@ -711,14 +737,29 @@ defmodule Plausible.Billing.QuotaTest do
now = NaiveDateTime.utc_now()
for site <- [site1, site2, site3] do
populate_stats(site, [
build(:event, timestamp: Timex.shift(now, days: -40), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -10), name: "custom"),
build(:event, timestamp: Timex.shift(now, days: -9), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -8), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -7), name: "pageview"),
build(:event, timestamp: Timex.shift(now, days: -6), name: "custom")
])
journey site, now: Timex.shift(now, days: -40) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -10) do
custom_event "custom"
end
journey site, now: Timex.shift(now, days: -9) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -8) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
journey site, now: Timex.shift(now, days: -6) do
custom_event "custom"
end
end
assert %{
@ -749,23 +790,29 @@ defmodule Plausible.Billing.QuotaTest do
user = insert(:user)
site = insert(:site, members: [user])
populate_stats(site, [
build(:event, timestamp: ~N[2023-04-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-04-05 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-05-01 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-02 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-03 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-04 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "pageview"),
build(:event, timestamp: ~N[2023-06-01 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-02 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-03 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-04 00:00:00], name: "custom"),
build(:event, timestamp: ~N[2023-06-05 00:00:00], name: "custom")
])
journey site, now: ~N[2023-04-01 00:00:00] do
custom_event "customx1", idle: &tomorrow/1
custom_event "customx2", idle: &tomorrow/1
custom_event "customx3", idle: &tomorrow/1
custom_event "customx4", idle: &tomorrow/1
custom_event "customx5"
end
journey site, now: ~N[2023-05-01 00:00:00] do
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
pageview "/", idle: &tomorrow/1
end
journey site, now: ~N[2023-06-01 00:00:00] do
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom", idle: &tomorrow/1
custom_event "custom"
end
{:ok, %{user: user}}
end
@ -807,9 +854,9 @@ defmodule Plausible.Billing.QuotaTest do
]
)
populate_stats(different_site, [
build(:event, timestamp: ~N[2023-05-05 00:00:00], name: "custom")
])
journey different_site, now: ~N[2023-05-05 00:00:00] do
custom_event "custom"
end
last_bill_date = ~D[2023-06-03]
today = ~D[2023-06-05]

@ -0,0 +1,63 @@
defmodule Plausible.FunnelsSamplingTest do
use Plausible.DataCase, async: false
@moduletag :full_build_only
use Plausible
use Plausible.Test.Support.Journey
# This test does a massive ingest, so we make sure the caches are utilized
# (both Sites.Cache and UA parsing). We also ingest on all cores, where only
# initially SipHash is a slow-down but warms up later on.
# On 24 cores it takes around 5.5s.
on_full_build do
alias Plausible.Goals
alias Plausible.Funnels
alias Plausible.Stats
setup do
site = insert(:site)
{:ok, g1} = Goals.create(site, %{"page_path" => "/go/to/blog/**"})
{:ok, g2} = Goals.create(site, %{"event_name" => "Signup"})
{:ok, g3} = Goals.create(site, %{"page_path" => "/checkout"})
{:ok,
%{
site: site,
goals: [g1, g2, g3],
steps: [g1, g2, g3] |> Enum.map(&%{"goal_id" => &1.id})
}}
end
setup_patch_env(Plausible.Cache, enabled: true)
test "sampling", %{steps: [g1, g2, g3]} do
site = insert(:site, domain: "1.1.1.1")
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup",
[g1, g2, g3]
)
Plausible.Site.Cache.refresh_all()
journey = MassJourney
journey site, manual: journey, ip: &random_ipv6/0, user_agent: "JourneyBrowser" do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey.run_many(50_000)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
{:ok, funnel_data} = Stats.funnel(site, query, funnel.id)
assert_in_delta funnel_data[:all_visitors], 50_000, 5000
end
end
end

@ -1,5 +1,5 @@
defmodule Plausible.FunnelsTest do
use Plausible.DataCase
use Plausible.DataCase, async: true
@moduletag :full_build_only
use Plausible
@ -171,14 +171,20 @@ defmodule Plausible.FunnelsTest do
]
)
populate_stats(site, [
build(:pageview, pathname: "/irrelevant/page/not/in/funnel", user_id: 999),
build(:pageview, pathname: "/go/to/blog/foo", user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, pathname: "/checkout", user_id: 123),
build(:pageview, pathname: "/go/to/blog/bar", user_id: 666),
build(:event, name: "Signup", user_id: 666)
])
journey site do
pageview "/irrelevant/page/not/in/funnel"
end
journey site, browser: Safari, debug: true do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey site, debug: true do
pageview "/go/to/blog/bar"
custom_event "Signup"
end
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
@ -231,13 +237,16 @@ defmodule Plausible.FunnelsTest do
[g1, g2, g3]
)
populate_stats(site, [
build(:pageview, pathname: "/go/to/blog/foo", user_id: 123),
build(:event, name: "Signup", user_id: 123),
build(:pageview, pathname: "/checkout", user_id: 123),
build(:pageview, pathname: "/go/to/blog/bar", user_id: 666),
build(:event, name: "Signup", user_id: 666)
])
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
pageview "/checkout"
end
journey site do
pageview "/go/to/blog/foo"
custom_event "Signup"
end
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
@ -328,36 +337,6 @@ defmodule Plausible.FunnelsTest do
]
}} = funnel_data
end
@tag :slow
test "sampling", %{site: site, steps: [g1, g2, g3 | _]} do
{:ok, funnel} =
Funnels.create(
site,
"From blog to signup and purchase",
[g1, g2, g3]
)
stats =
1..50_000
|> Enum.flat_map(fn _ ->
user_id = SipHash.hash!("0123456789ABCDEF", :crypto.strong_rand_bytes(64))
[
build(:pageview, pathname: "/go/to/blog/foo", user_id: user_id),
build(:event, name: "Signup", user_id: user_id),
build(:pageview, pathname: "/checkout", user_id: user_id)
]
end)
populate_stats(site, stats)
query =
Plausible.Stats.Query.from(site, %{"period" => "all", "sample_threshold" => "10000"})
{:ok, funnel_data} = Stats.funnel(site, query, funnel.id)
assert_in_delta funnel_data[:all_visitors], 50_000, 5000
end
end
end
end

@ -95,23 +95,11 @@ defmodule Plausible.PropsTest do
user = insert(:user, subscription: build(:growth_subscription))
site = insert(:site, members: [user])
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
assert {:error, :upgrade_required} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: nil} = Plausible.Repo.reload!(site)
@ -120,23 +108,11 @@ defmodule Plausible.PropsTest do
test "allow_existing_props/1 saves the most frequent prop keys" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, site} = Plausible.Props.allow_existing_props(site)
@ -144,74 +120,39 @@ defmodule Plausible.PropsTest do
Plausible.Repo.reload!(site)
end
test "allow_existing_props/1 skips invalid keys" do
site = insert(:site)
# test "allow_existing_props/1 skips invalid keys" do
# site = insert(:site)
# long = String.duplicate("a", 301)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", String.duplicate("a", 301)],
"meta.value": ["500", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
# journey site do
# custom_event "Payment", props: %{:amount => 500, long => true}
# custom_event "Payment", props: %{amount: 100, logged_in: false}
# custom_event "Payment", props: %{amount: 100, is_customer: false}
# end
{:ok, site} = Plausible.Props.allow_existing_props(site)
# {:ok, site} = Plausible.Props.allow_existing_props(site)
assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
Plausible.Repo.reload!(site)
end
# assert %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]} =
# Plausible.Repo.reload!(site)
# end
test "allow_existing_props/1 can be run multiple times" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount"],
"meta.value": ["500"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in"],
"meta.value": ["100", "false"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer"],
"meta.value": ["100", "false"]
)
])
journey site do
custom_event "Payment", props: %{:amount => 500}
custom_event "Payment", props: %{amount: 100, logged_in: false}
custom_event "Payment", props: %{amount: 100, is_customer: false}
end
{:ok, %Plausible.Site{allowed_event_props: ["amount", "logged_in", "is_customer"]}} =
Plausible.Props.allow_existing_props(site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
{:ok,
%Plausible.Site{
@ -229,23 +170,11 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 returns prop keys from events" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "Payment",
"meta.key": ["amount", "os"],
"meta.value": ["500", "Windows"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "logged_in", "with_error"],
"meta.value": ["100", "false", "true"]
),
build(:event,
name: "Payment",
"meta.key": ["amount", "is_customer", "first_time_customer"],
"meta.value": ["100", "false", "true"]
)
])
journey site do
custom_event "Payment", props: %{amount: 500, os: "Windows"}
custom_event "Payment", props: %{amount: 100, logged_in: false, with_error: true}
custom_event "Payment", props: %{amount: 100, is_customer: false, first_time_customer: true}
end
assert ["amount", "first_time_customer", "is_customer", "logged_in", "os", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()
@ -254,23 +183,14 @@ defmodule Plausible.PropsTest do
test "suggest_keys_to_allow/2 does not return internal prop keys from special event types" do
site = insert(:site)
populate_stats(site, [
build(:event,
name: "File Download",
"meta.key": ["url", "logged_in"],
"meta.value": ["http://file.test", "false"]
),
build(:event,
name: "Outbound Link: Click",
"meta.key": ["url", "first_time_customer"],
"meta.value": ["http://link.test", "true"]
),
build(:event,
name: "404",
"meta.key": ["path", "with_error"],
"meta.value": ["/i-dont-exist", "true"]
)
])
journey site do
custom_event "File Download", props: %{url: "http://file.test", logged_in: false}
custom_event "Outbound Link: Click",
props: %{url: "http://link.test", first_time_customer: true}
custom_event "404", props: %{path: "/i-dont-exist", with_error: true}
end
assert ["first_time_customer", "logged_in", "with_error"] ==
site |> Plausible.Props.suggest_keys_to_allow() |> Enum.sort()

@ -49,9 +49,9 @@ defmodule Plausible.SitesTest do
test "is date if site does have stats" do
site = insert(:site)
populate_stats(site, [
build(:pageview)
])
journey site do
pageview "/"
end
assert Sites.stats_start_date(site) == Timex.today(site.timezone)
end

@ -56,14 +56,18 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageview "/"
end
assert %{
change: 100,
@ -104,21 +108,31 @@ defmodule Plausible.Stats.ClickhouseTest do
site2 = insert(:site)
site3 = insert(:site)
user_id = 111
journey site1, now: ~N[2023-10-25 13:00:00] do
pageview "/", idle: 60
pageview "/"
end
populate_stats(site1, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 13:01:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
journey site1, now: ~N[2023-10-25 15:58:00] do
pageview "/"
end
populate_stats(site2, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:01:00])
])
journey site1, now: ~N[2023-10-25 15:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site2, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60
pageview "/"
end
assert result =
Clickhouse.last_24h_visitors_hourly_intervals([site1, site2, site3], fixed_now)
@ -144,13 +158,25 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-24 11:58:00]),
build(:pageview, timestamp: ~N[2023-10-24 12:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:59:00])
])
journey site, now: ~N[2023-10-24 11:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-24 12:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 50,
@ -164,10 +190,10 @@ defmodule Plausible.Stats.ClickhouseTest do
user_id = 111
populate_stats(site, [
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:59:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 15:59:00] do
pageview "/", idle: 60
pageview "/"
end
result = Clickhouse.last_24h_visitors_hourly_intervals([site], fixed_now)[site.domain]
assert result[:visitors] == 1
@ -177,14 +203,18 @@ defmodule Plausible.Stats.ClickhouseTest do
fixed_now = ~N[2023-10-26 10:00:15]
site = insert(:site)
user_id = 111
journey site, now: ~N[2023-10-25 15:00:00] do
pageview "/", idle: 60 * 60
pageview "/"
end
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 13:59:00]),
build(:pageview, timestamp: ~N[2023-10-25 13:58:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 15:00:00]),
build(:pageview, user_id: user_id, timestamp: ~N[2023-10-25 16:00:00])
])
journey site, now: ~N[2023-10-25 13:58:00] do
pageview "/"
end
journey site, now: ~N[2023-10-25 13:59:00] do
pageview "/"
end
assert %{
change: 100,
@ -225,27 +255,25 @@ defmodule Plausible.Stats.ClickhouseTest do
site = insert(:site)
query = Plausible.Stats.Query.from(site, %{"period" => "all"})
populate_stats(site, [
build(:pageview,
pathname: "/",
referrer_source: "Twitter"
),
build(:pageview,
pathname: "/plausible.io"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Google"
),
build(:pageview,
pathname: "/plausible.io",
referrer_source: "Bing"
)
])
journey site do
pageview "/", referrer: "https://twitter.com"
end
journey site do
pageview "/plausible.io"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://google.com"
end
journey site do
pageview "/plausible.io", referrer: "https://bing.com"
end
assert [
%{count: 2, name: "Google"},

@ -0,0 +1,247 @@
defmodule Plausible.Test.Support.Journey do
@moduledoc "TODO"
defmacro __using__(_) do
quote do
require Plausible.Test.Support.Journey
import Plausible.Test.Support.Journey
Module.register_attribute(__MODULE__, :journey_index, accumulate: false)
Module.put_attribute(__MODULE__, :journey_index, 0)
end
end
import Plug.Adapters.Test.Conn, only: [conn: 4]
import Plug.Conn
def run(site, state, journey) do
conn = new_conn(state)
# credo:disable-for-this-file Credo.Check.Warning.UnusedEnumOperation
Enum.reduce(journey, state, fn
{:pageview, [url, opts]}, state ->
payload =
%{
name: "pageview",
domain: site.domain,
url: build_url(site, url, opts)
}
|> add_common_body_params(opts)
conn
|> Plug.Adapters.Test.Conn.conn(:post, "/api/events", payload)
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
{:custom_event, [name, opts]}, state ->
payload =
%{name: name, domain: site.domain, url: build_url(site, "/", opts)}
|> add_common_body_params(opts)
conn
|> conn(:post, "/api/events", payload)
|> ingest(state, invoke_if_function(Keyword.get(opts, :idle, 1), state.now))
end)
if !state[:manual] do
flush_buffers()
end
end
defp add_common_body_params(payload, params) do
params =
params
|> Enum.into(%{})
|> Map.take(~w[revenue referrer url props]a)
if Enum.empty?(params) do
payload
else
Map.merge(payload, params)
end
end
def build_url(site, path, params) do
if params[:url] do
params[:url]
else
site_domain = site.domain |> URI.encode_www_form()
query_string =
params
|> Enum.into(%{})
|> Map.take(~w[utm_source utm_medium utm_campaign utm_term utm_content]a)
|> URI.encode_query()
uri = URI.new!("https://" <> Path.join(site_domain, path))
uri = %{uri | query: if(query_string != "", do: query_string)}
to_string(uri)
end
end
defp new_conn(state) do
(state.conn || %Plug.Conn{})
|> put_req_header("content-type", "application/json")
|> maybe_add_header("x-forwarded-for", invoke_if_function(state.ip))
|> prepare_device(state)
end
defp prepare_device(conn, state) do
case state[:browser] do
browser when is_atom(browser) and not is_nil(browser) ->
put_req_header(conn, "user-agent", sample_user_agent(browser))
nil ->
maybe_add_header(conn, "user-agent", invoke_if_function(state.user_agent))
end
end
defp sample_user_agent(Safari) do
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1"
end
defp sample_user_agent(Edge) do
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0."
end
defp sample_user_agent(x) do
raise "User agent #{x} unsupported, add me :)"
end
defp maybe_add_header(conn, _header, nil) do
conn
end
defp maybe_add_header(conn, header, value) do
put_req_header(conn, header, value)
end
defp invoke_if_function(f) when is_function(f, 0), do: f.()
defp invoke_if_function(value), do: value
defp invoke_if_function(f, arg) when is_function(f, 1), do: f.(arg)
defp invoke_if_function(value, _), do: value
def flush_buffers do
Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush()
end
defp ingest(conn, state, idle) do
now =
state.now
|> invoke_if_function()
|> Plausible.Timezones.to_utc_datetime()
idle_offset =
case idle do
n when is_integer(n) ->
n
%NaiveDateTime{} = ndt ->
NaiveDateTime.diff(ndt, now, :second)
end
{:ok, request} =
Plausible.Ingestion.Request.build(conn, now)
Plausible.Ingestion.Event.build_and_buffer(request)
new_now = NaiveDateTime.add(now, idle_offset, :second)
if state.debug do
debug(conn, state, idle, now, new_now)
end
Map.put(state, :now, new_now)
end
defp debug(conn, _state, idle, now, new_now) do
IO.puts("\n[#{now}] Request:" <> IO.ANSI.yellow())
IO.puts(" #{conn.method} #{conn.request_path}?#{conn.query_string}" <> IO.ANSI.cyan())
IO.puts(" user-agent: #{get_req_header(conn, "user-agent")}")
IO.puts(" x-forwarded-for: #{get_req_header(conn, "x-forwarded-for")}" <> IO.ANSI.green())
IO.puts(" #{Jason.encode!(conn.body_params)}" <> IO.ANSI.reset())
IO.puts("[#{new_now}] Idle complete (#{inspect(idle)})")
end
defmacro default(state) do
h1 = :erlang.phash2(__CALLER__.module, 256)
h2 = :erlang.phash2(__CALLER__.line, 256)
default_ip = "#{h1}.#{h2}.#{h1}.#{h2}"
default_user_agent = "JourneyBrowser #{__CALLER__.module}/#{__CALLER__.line}"
quote do
unquote(state)
|> Map.update(:now, &NaiveDateTime.utc_now/0, & &1)
|> Map.update(:conn, nil, & &1)
|> Map.update(:ip, unquote(default_ip), & &1)
|> Map.update(:user_agent, unquote(default_user_agent), & &1)
|> Map.update(:debug, false, & &1)
end
end
defmacro journey(site, state \\ [], do: block) do
Code.compiler_options(ignore_module_conflict: true)
idx = Module.get_attribute(__CALLER__.module, :journey_index)
idx = idx + 1
Module.put_attribute(__CALLER__.module, :journey_index, idx)
mod = :"#{__CALLER__.module}.Journey#{idx}"
__journey__(aliased(state[:manual]) || mod, site, state, block)
end
defp aliased({:__aliases__, _, mod}), do: Module.concat(mod)
defp aliased(other), do: other
defp __journey__(mod, site, state, block) do
quote do
defmodule unquote(mod) do
Module.register_attribute(__MODULE__, :journey, accumulate: true)
@site unquote(site)
@initial_state default(Enum.into(unquote(state), %{}))
unquote(block)
def run() do
Plausible.Test.Support.Journey.run(
@site,
@initial_state,
Enum.reverse(@journey)
)
end
def flush() do
Plausible.Test.Support.Journey.flush_buffers()
end
def run_many(n) do
1..n
|> Task.async_stream(fn _ -> run() end)
|> Stream.run()
flush()
end
end
if is_nil(unquote(state)[:manual]) do
unquote(mod).run()
end
end
end
defmacro pageview(url, opts \\ []) do
quote do: store(:pageview, [unquote(url), unquote(opts)])
end
defmacro custom_event(name, opts \\ []) do
quote do: store(:custom_event, [unquote(name), unquote(opts)])
end
defmacro store(op, args) do
quote do
Module.put_attribute(__MODULE__, :journey, {unquote(op), unquote(args)})
end
end
end

@ -2,10 +2,13 @@ defmodule Plausible.TestUtils do
use Plausible.Repo
alias Plausible.Factory
require Logger
defmacro __using__(_) do
quote do
require Plausible.TestUtils
import Plausible.TestUtils
use Plausible.Test.Support.Journey
end
end
@ -257,4 +260,34 @@ defmodule Plausible.TestUtils do
def random_ip() do
Enum.map_join(1..4, ".", fn _ -> Enum.random(1..254) end)
end
def random_ipv6() do
<<a1::16, a2::16, a3::16, a4::16, a5::16, a6::16, a7::16, a8::16>> =
:crypto.strong_rand_bytes(16)
[a1, a2, a3, a4, a5, a6, a7, a8]
|> Enum.map_join(":", &Base.encode16(<<&1>>, case: :lower))
end
def tomorrow(now) do
NaiveDateTime.add(now, 1, :day)
end
def rand_user_agent() do
[
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_0_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/156.1.368858699 Mobile/18A393 Safari/604.1",
"Mozilla/5.0 (Linux; Android 9; SM-A305G Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.82 Mobile Safari/537.36 GSA/12.14.7.23.arm64",
"Mozilla/5.0 (Linux; Android 10; moto g(8) plus Build/QPIS30.28-Q3-28-26-3-4; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.66 Mobile Safari/537.36 GSA/12.14.7.23.arm64",
"com.google.GoogleMobile/156.1 iPhone/14.0.1 hw/iPhone10_4",
"Mozilla/5.0 (Linux; Android 10; vivo 1819 Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.66 Mobile Safari/537.36 GSA/12.14.7.23.arm64",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.2; Win64; x64; Trident/7.0; .NET4.0C; .NET4.0E; InfoPath.3; Tablet PC 2.0; HCTE; ms-office; MSOffice 15)",
"Mozilla/5.0 (Linux; Android 5.1; F5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.181 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 9; LM-X420 Build/PKQ1.190522.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/90.0.4430.66 Mobile Safari/537.36 GSA/12.14.7.23.arm",
"Mozilla/5.0 (Linux; Android 10; SM-A516B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.82 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 7.0; Mi-4c) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.66 Mobile Safari/537.36"
]
|> Enum.random()
end
def rand_idle(_), do: Enum.random(1..100)
end

@ -26,14 +26,16 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
end
test "does not send any notifications when site has stats older than 2d" do
today = Date.utc_today()
next_week = today |> Date.add(+7)
today = NaiveDateTime.utc_now()
next_week = today |> NaiveDateTime.add(+7, :day)
user = insert(:user, accept_traffic_until: next_week)
:site
|> insert(members: [user])
|> populate_stats([build(:pageview, timestamp: Date.add(today, -3))])
|> journey now: NaiveDateTime.add(today, -3, :day) do
pageview "/"
end
{:ok, 1} = AcceptTrafficUntil.perform(nil)
@ -41,15 +43,17 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
end
test "does send notification when last stat is 2d old" do
today = Date.utc_today()
next_week = today |> Date.add(+7)
today = NaiveDateTime.utc_now()
next_week = today |> NaiveDateTime.add(+7, :day)
user =
insert(:user, accept_traffic_until: next_week)
:site
|> insert(members: [user])
|> populate_stats([build(:pageview, timestamp: Date.add(today, -2))])
|> journey now: NaiveDateTime.add(today, -2, :day) do
pageview "/"
end
{:ok, 1} = AcceptTrafficUntil.perform(nil)
@ -60,7 +64,11 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
tomorrow = Date.utc_today() |> Date.add(+1)
user = insert(:user, accept_traffic_until: tomorrow)
:site |> insert(members: [user]) |> populate_stats([build(:pageview)])
:site
|> insert(members: [user])
|> journey do
pageview "/"
end
{:ok, 1} = AcceptTrafficUntil.perform(nil)
assert_final_notification(user.email)
@ -70,7 +78,7 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
next_week = Date.utc_today() |> Date.add(+7)
user = insert(:user, accept_traffic_until: next_week)
:site |> insert(members: [user]) |> populate_stats([build(:pageview)])
:site |> insert(members: [user]) |> journey(do: pageview("/"))
{:ok, 1} = AcceptTrafficUntil.perform(nil)
assert_weekly_notification(user.email)
@ -87,10 +95,10 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
user3 = insert(:user, accept_traffic_until: in_8_days, email: "nope@example.com")
user4 = insert(:user, accept_traffic_until: today, email: "nope2@example.com")
:site |> insert(members: [user1]) |> populate_stats([build(:pageview)])
:site |> insert(members: [user2]) |> populate_stats([build(:pageview)])
:site |> insert(members: [user3]) |> populate_stats([build(:pageview)])
:site |> insert(members: [user4]) |> populate_stats([build(:pageview)])
:site |> insert(members: [user1]) |> journey(do: pageview("/"))
:site |> insert(members: [user2]) |> journey(do: pageview("/"))
:site |> insert(members: [user3]) |> journey(do: pageview("/"))
:site |> insert(members: [user4]) |> journey(do: pageview("/"))
{:ok, 2} = AcceptTrafficUntil.perform(nil)
assert_weekly_notification(user1.email)
@ -105,12 +113,21 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do
user = insert(:user, email: email) |> Plausible.Auth.User.start_trial() |> Repo.update!()
site = insert(:site, members: [user])
populate_stats(site, [
build(:pageview, timestamp: today),
build(:pageview, timestamp: Date.add(user.trial_expiry_date, 7)),
build(:pageview, timestamp: Date.add(user.trial_expiry_date, 8)),
build(:pageview, timestamp: Date.add(user.trial_expiry_date, 13))
])
journey site do
pageview "/"
end
journey site, now: Date.add(user.trial_expiry_date, 7) do
pageview "/"
end
journey site, now: Date.add(user.trial_expiry_date, 8) do
pageview "/"
end
journey site, now: Date.add(user.trial_expiry_date, 13) do
pageview "/"
end
# today's worker is no-op
{:ok, 0} = AcceptTrafficUntil.perform(nil, today)

@ -77,15 +77,14 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, domain: "test-site.com", inserted_at: Timex.shift(now, days: -8))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview,
user_id: 123,
timestamp: Timex.shift(now, days: -7),
referrer_source: "Google"
),
build(:pageview, user_id: 123, timestamp: Timex.shift(now, days: -7)),
build(:pageview, timestamp: Timex.shift(now, days: -7))
])
journey site, now: Timex.shift(now, days: -7) do
pageview "/", referrer: "https://google.com"
pageview "/"
end
journey site, now: Timex.shift(now, days: -7) do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -110,12 +109,18 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, user_id: 1, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago),
build(:pageview, user_id: 2, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -145,12 +150,18 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, user_id: 1, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, user_id: 2, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: two_weeks_ago do
pageview "/"
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})
@ -180,10 +191,13 @@ defmodule Plausible.Workers.SendEmailReportTest do
site = insert(:site, inserted_at: Timex.shift(now, days: -15))
insert(:weekly_report, site: site, recipients: ["user@email.com"])
populate_stats(site, [
build(:pageview, timestamp: two_weeks_ago),
build(:pageview, timestamp: week_ago)
])
journey site, now: two_weeks_ago do
pageview "/"
end
journey site, now: week_ago do
pageview "/"
end
perform_job(SendEmailReport, %{"site_id" => site.id, "interval" => "weekly"})

Loading…
Cancel
Save