Merge branch 'master' into visitor-graph

simplify
RobertJoonas 3 years ago committed by GitHub
commit 40b42ad4f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -0,0 +1,28 @@
name: Validate migrations
on:
pull_request:
paths:
- 'priv/repo/migrations/**'
- 'priv/ingest_repo/migrations/**'
jobs:
validate:
name: App code does not change at the same time
runs-on: ubuntu-latest
steps:
- uses: dorny/paths-filter@v3
id: changes
with:
list-files: json
filters: |
lib:
- 'lib/**'
- 'extra/**'
- 'config/**'
- if: steps.changes.outputs.lib == 'true'
run: |
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
exit 1

@ -2,6 +2,8 @@
All notable changes to this project will be documented in this file.
### Added
- Pages Block List in Site Settings
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export
- Ability to display total conversions (with a goal filter) on the main graph
- Add `conversion_rate` to Stats API Timeseries and on the main graph
@ -55,6 +57,8 @@ All notable changes to this project will be documented in this file.
- Validate metric isn't queried multiple times
### Fixed
- Creating many sites no longer leads to cookie overflow
- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns
- Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown
- Fix `conversion_rate` metric in a `browser_versions` breakdown
- Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division)

@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) {
})
}
}
let Uploaders = {}
Uploaders.S3 = function (entries, onViewError) {
entries.forEach(entry => {
let xhr = new XMLHttpRequest()
onViewError(() => xhr.abort())
xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
xhr.onerror = () => entry.error()
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
let percent = Math.round((event.loaded / event.total) * 100)
if (percent < 100) { entry.progress(percent) }
}
})
let url = entry.meta.url
xhr.open("PUT", url, true)
xhr.send(entry.file)
})
}
let token = csrfToken.getAttribute("content")
let url = websocketUrl.getAttribute("content")
let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
let liveSocket = new LiveSocket(liveUrl, Socket, {
heartbeatIntervalMs: 10000,
params: { _csrf_token: token }, hooks: Hooks, dom: {
params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {
// for alpinejs integration
onBeforeElUpdated(from, to) {
if (from._x_dataStack) {

@ -77,6 +77,21 @@ defmodule Plausible.Application do
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.All,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)},

@ -176,26 +176,16 @@ defmodule Plausible.Cache do
defp refresh(mode, query, opts) when mode in @modes do
cache_name = Keyword.get(opts, :cache_name, name())
measure_duration(telemetry_event_refresh(cache_name, mode), fn ->
items = Plausible.Repo.all(query)
:ok = merge_items(items, opts)
end)
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(
telemetry_event_refresh(cache_name, mode),
fn ->
items = Plausible.Repo.all(query)
:ok = merge_items(items, opts)
end
)
:ok
end
defp measure_duration(event, fun) when is_function(fun, 0) do
{duration, result} = time_it(fun)
:telemetry.execute(event, %{duration: duration}, %{})
result
end
defp time_it(fun) do
start = System.monotonic_time()
result = fun.()
stop = System.monotonic_time()
{stop - start, result}
end
end
end
end

@ -12,14 +12,24 @@ defmodule Plausible.Exports do
Examples:
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
"plausible.io_20210101_20241231.zip"
"plausible_io_20210101_20241231.zip"
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
"Bücher.example_20210101_20241231.zip"
"Bücher_example_20210101_20241231.zip"
"""
def archive_filename(domain, min_date, max_date) do
"#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip"
name =
Enum.join(
[
String.replace(domain, ".", "_"),
Calendar.strftime(min_date, "%Y%m%d"),
Calendar.strftime(max_date, "%Y%m%d")
],
"_"
)
name <> ".zip"
end
@doc """

@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do
Date.range(~D[2019-01-01], ~D[2022-01-01])
iex> date_range([])
** (ArgumentError) empty uploads
nil
"""
@spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()
@spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
def date_range([]), do: raise(ArgumentError, "empty uploads")
def date_range([]), do: nil
defp date_range([upload | uploads], prev_start_date, prev_end_date) do
filename =
@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter do
def parse_filename!(_filename) do
raise ArgumentError, "invalid filename"
end
@doc """
Checks if the provided filename conforms to the expected format.
Examples:
iex> valid_filename?("my_data.csv")
false
iex> valid_filename?("imported_devices_00010101_20250101.csv")
true
"""
@spec valid_filename?(String.t()) :: boolean
def valid_filename?(filename) do
try do
parse_filename!(filename)
else
_ -> true
rescue
_ -> false
end
end
@doc """
Extracts the table name from the provided filename.
Raises if the filename doesn't conform to the expected format.
Examples:
iex> extract_table("my_data.csv")
** (ArgumentError) invalid filename
iex> extract_table("imported_devices_00010101_20250101.csv")
"imported_devices"
"""
@spec extract_table(String.t()) :: String.t()
def extract_table(filename) do
{table, _start_date, _end_date} = parse_filename!(filename)
table
end
end

@ -29,6 +29,7 @@ defmodule Plausible.Ingestion.Event do
| :dc_ip
| :site_ip_blocklist
| :site_country_blocklist
| :site_page_blocklist
@type t() :: %__MODULE__{
domain: String.t() | nil,
@ -99,6 +100,10 @@ defmodule Plausible.Ingestion.Event do
[:plausible, :ingest, :event, :dropped]
end
def telemetry_pipeline_step_duration() do
[:plausible, :ingest, :pipeline, :step]
end
@spec emit_telemetry_buffered(t()) :: :ok
def emit_telemetry_buffered(event) do
:telemetry.execute(telemetry_event_buffered(), %{}, %{
@ -118,33 +123,42 @@ defmodule Plausible.Ingestion.Event do
defp pipeline() do
[
&drop_datacenter_ip/1,
&drop_shield_rule_ip/1,
&put_geolocation/1,
&drop_shield_rule_country/1,
&put_user_agent/1,
&put_basic_info/1,
&put_referrer/1,
&put_utm_tags/1,
&put_props/1,
&put_revenue/1,
&put_salts/1,
&put_user_id/1,
&validate_clickhouse_event/1,
&register_session/1,
&write_to_buffer/1
drop_datacenter_ip: &drop_datacenter_ip/1,
drop_shield_rule_page: &drop_shield_rule_page/1,
drop_shield_rule_ip: &drop_shield_rule_ip/1,
put_geolocation: &put_geolocation/1,
drop_shield_rule_country: &drop_shield_rule_country/1,
put_user_agent: &put_user_agent/1,
put_basic_info: &put_basic_info/1,
put_referrer: &put_referrer/1,
put_utm_tags: &put_utm_tags/1,
put_props: &put_props/1,
put_revenue: &put_revenue/1,
put_salts: &put_salts/1,
put_user_id: &put_user_id/1,
validate_clickhouse_event: &validate_clickhouse_event/1,
register_session: &register_session/1,
write_to_buffer: &write_to_buffer/1
]
end
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
case pipeline_step.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event ->
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(
telemetry_pipeline_step_duration(),
fn -> execute_step(step_fn, acc_event) end,
%{step: "#{step_name}"}
)
end)
end
defp execute_step(step_fn, acc_event) do
case step_fn.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end
defp new(domain, request) do
struct!(__MODULE__, domain: domain, request: request)
end
@ -182,15 +196,18 @@ defmodule Plausible.Ingestion.Event do
end
defp drop_shield_rule_ip(%__MODULE__{} = event) do
domain = event.domain
address = event.request.remote_ip
case Plausible.Shield.IPRuleCache.get({domain, address}) do
%Plausible.Shield.IPRule{action: :deny} ->
drop(event, :site_ip_blocklist)
if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do
drop(event, :site_ip_blocklist)
else
event
end
end
_ ->
event
defp drop_shield_rule_page(%__MODULE__{} = event) do
if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do
drop(event, :site_page_blocklist)
else
event
end
end
@ -263,12 +280,10 @@ defmodule Plausible.Ingestion.Event do
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
)
when is_binary(domain) and is_binary(cc) do
case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
%Plausible.Shield.CountryRule{action: :deny} ->
drop(event, :site_country_blocklist)
_ ->
event
if Plausible.Shields.country_blocked?(domain, cc) do
drop(event, :site_country_blocklist)
else
event
end
end

@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do
|> cast(attrs, [:site_id, :country_code])
|> validate_required([:site_id, :country_code])
|> validate_length(:country_code, is: 2)
|> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc ->
|> validate_change(:country_code, fn :country_code, cc ->
if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do
[]
else

@ -0,0 +1,85 @@
defmodule CompiledRegex do
@moduledoc """
Ensures that the regex is compiled on load
"""
use Ecto.Type
def type, do: :string
def cast(val) when is_binary(val), do: {:ok, val}
def cast(_), do: :error
def load(val), do: {:ok, Regex.compile!(val)}
def dump(val), do: {:ok, val}
end
defmodule Plausible.Shield.PageRule do
@moduledoc """
Schema for Pages block list
"""
use Ecto.Schema
import Ecto.Changeset
@type t() :: %__MODULE__{}
@primary_key {:id, :binary_id, autogenerate: true}
schema "shield_rules_page" do
belongs_to :site, Plausible.Site
field :page_path, :string
field :page_path_pattern, CompiledRegex
field :action, Ecto.Enum, values: [:deny, :allow], default: :deny
field :added_by, :string
# If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache`
field :from_cache?, :boolean, virtual: true, default: false
timestamps()
end
def changeset(rule \\ %__MODULE__{}, attrs) do
rule
|> cast(attrs, [:site_id, :page_path])
|> validate_required([:site_id, :page_path])
|> validate_length(:page_path, max: 250)
|> validate_change(:page_path, fn :page_path, p ->
if not String.starts_with?(p, "/") do
[page_path: "must start with /"]
else
[]
end
end)
|> store_regex()
|> unique_constraint(:page_path_pattern,
name: :shield_rules_page_site_id_page_path_pattern_index,
error_key: :page_path,
message: "rule already exists"
)
end
defp store_regex(changeset) do
case get_field(changeset, :page_path) do
"/" <> _ = page_path ->
regex =
page_path
|> Regex.escape()
|> String.replace("\\*\\*", ".*")
|> String.replace("\\*", ".*")
regex = "^#{regex}$"
verify_valid_regex(changeset, regex)
_ ->
changeset
end
end
defp verify_valid_regex(changeset, regex) do
case Regex.compile(regex) do
{:ok, _} ->
put_change(changeset, :page_path_pattern, regex)
{:error, _} ->
add_error(changeset, :page_path, "could not compile regular expression")
end
end
end

@ -0,0 +1,65 @@
defmodule Plausible.Shield.PageRuleCache do
@moduledoc """
Allows retrieving Page Rules by domain
"""
alias Plausible.Shield.PageRule
import Ecto.Query
use Plausible.Cache
@cache_name :page_blocklist_by_domain
@cached_schema_fields ~w(
id
page_path_pattern
action
)a
@impl true
def name(), do: @cache_name
@impl true
def child_id(), do: :cache_page_blocklist
@impl true
def count_all() do
Plausible.Repo.aggregate(PageRule, :count)
end
@impl true
def base_db_query() do
from rule in PageRule,
inner_join: s in assoc(rule, :site),
select: {
s.domain,
s.domain_changed_from,
%{struct(rule, ^@cached_schema_fields) | from_cache?: true}
}
end
@impl true
def get_from_source(domain) do
query =
base_db_query()
|> where([..., site], site.domain == ^domain)
case Plausible.Repo.one(query) do
{_, _, rule} -> %PageRule{rule | from_cache?: false}
_any -> nil
end
end
@impl true
def unwrap_cache_keys(items) do
Enum.reduce(items, [], fn
{domain, nil, object}, acc ->
[{domain, object} | acc]
{domain, domain_changed_from, object}, acc ->
[
{domain, object},
{domain_changed_from, object} | acc
]
end)
end
end

@ -13,11 +13,63 @@ defmodule Plausible.Shields do
@maximum_country_rules 30
def maximum_country_rules(), do: @maximum_country_rules
@maximum_page_rules 30
def maximum_page_rules(), do: @maximum_page_rules
@spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()]
def list_ip_rules(site_or_id) do
list(Shield.IPRule, site_or_id)
end
@spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def ip_blocked?(%Site{domain: domain}, address) do
ip_blocked?(domain, address)
end
def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do
case Shield.IPRuleCache.get({domain, address}) do
%Shield.IPRule{action: :deny} ->
true
_ ->
false
end
end
@spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def page_blocked?(%Site{domain: domain}, address) do
page_blocked?(domain, address)
end
def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do
page_rules = Shield.PageRuleCache.get(domain)
if page_rules do
page_rules
|> List.wrap()
|> Enum.find_value(false, fn rule ->
rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname)
end)
else
false
end
end
@spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def country_blocked?(%Site{domain: domain}, country_code) do
country_blocked?(domain, country_code)
end
def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do
case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do
%Shield.CountryRule{action: :deny} ->
true
_ ->
false
end
end
@spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()}
def add_ip_rule(site_or_id, params, opts \\ []) do
@ -59,6 +111,28 @@ defmodule Plausible.Shields do
count(Shield.CountryRule, site_or_id)
end
@spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()]
def list_page_rules(site_or_id) do
list(Shield.PageRule, site_or_id)
end
@spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()}
def add_page_rule(site_or_id, params, opts \\ []) do
opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules})
add(Shield.PageRule, site_or_id, params, opts)
end
@spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok
def remove_page_rule(site_or_id, rule_id) do
remove(Shield.PageRule, site_or_id, rule_id)
end
@spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer()
def count_page_rules(site_or_id) do
count(Shield.PageRule, site_or_id)
end
defp list(schema, %Site{id: id}) do
list(schema, id)
end

@ -0,0 +1,136 @@
defmodule Plausible.Shields.CountryTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_country_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{})
assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported country", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"})
assert changeset.errors == [country_code: {"is invalid", []}]
refute changeset.valid?
end
test "incorrect country format", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"})
assert changeset.errors ==
[
{:country_code, {"is invalid", []}},
{:country_code,
{"should be %{count} character(s)",
[count: 2, validation: :length, kind: :is, type: :string]}}
]
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"})
refute changeset.valid?
assert changeset.errors == [
country_code:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_country_site_id_country_code_index"}
]}
]
end
test "over limit", %{site: site} do
country_codes =
Location.Country.all()
|> Enum.take(Plausible.Shields.maximum_country_rules())
|> Enum.map(& &1.alpha_2)
for cc <- country_codes do
assert {:ok, _} =
add_country_rule(site, %{"country_code" => cc})
end
assert count_country_rules(site) == maximum_country_rules()
assert {:error, changeset} =
add_country_rule(site, %{"country_code" => "US"})
refute changeset.valid?
assert changeset.errors == [country_code: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_country_rule(site, %{"country_code" => "EE"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_country_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_country_rule(site, %{"country_code" => "EE"})
assert remove_country_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.CountryRule, rule.id)
assert remove_country_rule(site, rule.id) == :ok
end
end
describe "list_country_rules/1" do
test "empty", %{site: site} do
assert(list_country_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_country_rule(site, %{"country_code" => "EE"})
:timer.sleep(1000)
{:ok, r2} = add_country_rule(site, %{"country_code" => "PL"})
assert [^r2, ^r1] = list_country_rules(site)
end
end
describe "count_country_rules/1" do
test "counts", %{site: site} do
assert count_country_rules(site) == 0
{:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert count_country_rules(site) == 1
{:ok, _} = add_country_rule(site, %{"country_code" => "PL"})
assert count_country_rules(site) == 2
end
end
describe "Country Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_country_rules(site.id) == 0
assert list_country_rules(site.id) == []
assert {:ok, rule} =
add_country_rule(site.id, %{"country_code" => "EE"})
add_country_rule(site2, %{"country_code" => "PL"})
assert count_country_rules(site) == 1
assert [^rule] = list_country_rules(site)
assert rule.country_code == "EE"
assert rule.action == :deny
refute rule.from_cache?
assert country_blocked?(site, "ee")
refute country_blocked?(site, "xx")
end
end
end

@ -0,0 +1,147 @@
defmodule Plausible.Shields.IPTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_ip_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{})
assert changeset.errors == [inet: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported netmask", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"})
assert changeset.errors == [inet: {"netmask unsupported", []}]
refute changeset.valid?
assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"})
end
test "incorrect ip", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"})
assert changeset.errors == [
inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]}
]
refute changeset.valid?
end
test "non-strict IPs", %{site: site} do
assert {:error, _} = add_ip_rule(site, %{"inet" => "111"})
end
test "double insert", %{site: site} do
assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
refute changeset.valid?
assert changeset.errors == [
inet:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_ip_site_id_inet_index"}
]}
]
end
test "ipv6", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "ipv4", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "over limit", %{site: site} do
for i <- 1..maximum_ip_rules() do
assert {:ok, _} =
add_ip_rule(site, %{"inet" => "1.1.1.#{i}"})
end
assert count_ip_rules(site) == maximum_ip_rules()
assert {:error, changeset} =
add_ip_rule(site, %{"inet" => "1.1.1.31"})
refute changeset.valid?
assert changeset.errors == [inet: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_ip_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert remove_ip_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.IPRule, rule.id)
assert remove_ip_rule(site, rule.id) == :ok
end
end
describe "list_ip_rules/1" do
test "empty", %{site: site} do
assert(list_ip_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
:timer.sleep(1000)
{:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert [^r2, ^r1] = list_ip_rules(site)
end
end
describe "count_ip_rules/1" do
test "counts", %{site: site} do
assert count_ip_rules(site) == 0
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert count_ip_rules(site) == 1
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert count_ip_rules(site) == 2
end
end
describe "IP Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_ip_rules(site.id) == 0
assert list_ip_rules(site.id) == []
assert {:ok, rule} =
add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"})
add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"})
assert count_ip_rules(site) == 1
assert [^rule] = list_ip_rules(site)
assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32}
assert rule.description == "Localhost"
assert rule.action == :deny
refute rule.from_cache?
assert ip_blocked?(site, "127.0.0.1")
refute ip_blocked?(site, "127.0.0.2")
end
end
end

@ -0,0 +1,174 @@
defmodule Plausible.Shields.PageTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_page_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{})
assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "no slash", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"})
assert changeset.errors == [page_path: {"must start with /", []}]
refute changeset.valid?
end
test "lengthy", %{site: site} do
long = "/" <> :binary.copy("a", 251)
assert {:error, changeset} = add_page_rule(site, %{"page_path" => long})
assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"})
refute changeset.valid?
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "equivalent rules are counted as dupes", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"})
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "regex storage: wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"})
assert rule.page_path_pattern == "^/test/.*$"
end
test "regex storage: no wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert rule.page_path_pattern == "^/test$"
end
test "regex storage: escaping", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"})
assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$"
end
test "over limit", %{site: site} do
for i <- 1..maximum_page_rules() do
assert {:ok, _} =
add_page_rule(site, %{"page_path" => "/test/#{i}"})
end
assert count_page_rules(site) == maximum_page_rules()
assert {:error, changeset} =
add_page_rule(site, %{"page_path" => "/test/31"})
refute changeset.valid?
assert changeset.errors == [page_path: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_page_rule(site, %{"page_path" => "/test"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "page pattern matching" do
test "no wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
end
test "wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
refute page_blocked?(site, "/test")
assert page_blocked?(site, "/test/")
assert page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
refute page_blocked?(site, "/testing")
end
end
describe "remove_page_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert remove_page_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.PageRule, rule.id)
assert remove_page_rule(site, rule.id) == :ok
end
end
describe "list_page_rules/1" do
test "empty", %{site: site} do
assert(list_page_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_page_rule(site, %{"page_path" => "/test"})
:timer.sleep(1000)
{:ok, r2} = add_page_rule(site, %{"page_path" => "/test"})
assert [^r2, ^r1] = list_page_rules(site)
end
end
describe "count_page_rules/1" do
test "counts", %{site: site} do
assert count_page_rules(site) == 0
{:ok, _} = add_page_rule(site, %{"page_path" => "/test1"})
assert count_page_rules(site) == 1
{:ok, _} = add_page_rule(site, %{"page_path" => "/test2"})
assert count_page_rules(site) == 2
end
end
describe "Page Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_page_rules(site.id) == 0
assert list_page_rules(site.id) == []
assert {:ok, rule} =
add_page_rule(site.id, %{
"page_path" => "/test"
})
add_page_rule(site2, %{"page_path" => "/test"})
assert count_page_rules(site) == 1
assert [%{id: rule_id}] = list_page_rules(site)
assert rule.id == rule_id
assert rule.page_path == "/test"
assert rule.action == :deny
refute rule.from_cache?
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/testing")
end
end
end

@ -529,6 +529,8 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:entry_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.entry_page,
select_merge: %{entry_page: s.entry_page},
order_by: {:asc, s.entry_page}
@ -538,6 +540,8 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:exit_page") do
from(
s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.exit_page,
select_merge: %{exit_page: s.exit_page},
order_by: {:asc, s.exit_page}

@ -47,6 +47,16 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
unit: {:native, :millisecond},
measurement: :duration
),
distribution(
metric_prefix ++ [:ingest, :events, :pipeline, :steps],
event_name: Ingestion.Event.telemetry_pipeline_step_duration(),
reporter_options: [
buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000]
],
unit: {:native, :microsecond},
measurement: :duration,
tags: [:step]
),
counter(
metric_prefix ++ [:ingest, :events, :buffered, :total],
event_name: Ingestion.Event.telemetry_event_buffered()
@ -110,6 +120,19 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
})
end
def measure_duration(event, fun, meta \\ %{}) when is_function(fun, 0) do
{duration, result} = time_it(fun)
:telemetry.execute(event, %{duration: duration}, meta)
result
end
defp time_it(fun) do
start = System.monotonic_time()
result = fun.()
stop = System.monotonic_time()
{stop - start, result}
end
defp write_buffer_metrics(metric_prefix, poll_rate) do
Polling.build(
:write_buffer_metrics,

@ -0,0 +1,50 @@
defmodule PlausibleWeb.Components.FirstDashboardLaunchBanner do
@moduledoc """
A banner that appears on the first dashboard launch
"""
use Phoenix.Component
use Phoenix.HTML
attr(:site, Plausible.Site, required: true)
def set(assigns) do
~H"""
<script>
sessionStorage.setItem('<%= storage_key(@site) %>', false);
</script>
"""
end
attr(:site, Plausible.Site, required: true)
def render(assigns) do
~H"""
<div
x-cloak
x-data={x_data(@site)}
class="w-full px-4 text-sm font-bold text-center text-blue-900 bg-blue-200 rounded transition"
style="top: 91px"
role="alert"
x-bind:class="! show ? 'hidden' : ''"
x-init={x_init(@site)}
>
<%= link("Team members, email reports and GA import. Explore more →",
to: "/#{URI.encode_www_form(@site.domain)}/settings/email-reports",
class: "py-2 block"
) %>
</div>
"""
end
defp x_data(site) do
"{show: !!sessionStorage.getItem('#{storage_key(site)}')}"
end
defp x_init(site) do
"setTimeout(() => sessionStorage.removeItem('#{storage_key(site)}'), 3000)"
end
defp storage_key(site) do
"dashboard_seen_#{site.domain}"
end
end

@ -34,9 +34,9 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Mailer.send()
end
conn
|> put_session(site.domain <> "_offer_email_report", true)
|> redirect(external: Routes.site_path(conn, :add_snippet, site.domain))
redirect(conn,
external: Routes.site_path(conn, :add_snippet, site.domain, site_created: true)
)
{:error, {:over_limit, limit}} ->
render(conn, "new.html",
@ -257,7 +257,7 @@ defmodule PlausibleWeb.SiteController do
end
def settings_shields(conn, %{"shield" => shield})
when shield in ["ip_addresses", "countries"] do
when shield in ["ip_addresses", "countries", "pages"] do
site = conn.assigns.site
conn
@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end
def export(conn, _params) do
def csv_export(conn, _params) do
%{site: site, current_user: user} = conn.assigns
Oban.insert!(
@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do
|> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))
end
def csv_import(conn, _params) do
conn
|> assign(:skip_plausible_tracking, true)
|> render("csv_import.html",
layout: {PlausibleWeb.LayoutView, "focus.html"},
connect_live_socket: true
)
end
def change_domain(conn, _params) do
changeset = Plausible.Site.update_changeset(conn.assigns.site)

@ -59,10 +59,7 @@ defmodule PlausibleWeb.StatsController do
cond do
stats_start_date && can_see_stats? ->
offer_email_report = get_session(conn, site.domain <> "_offer_email_report")
conn
|> remove_email_report_banner(site)
|> put_resp_header("x-robots-tag", "noindex, nofollow")
|> render("stats.html",
site: site,
@ -72,7 +69,6 @@ defmodule PlausibleWeb.StatsController do
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
title: title(conn, site),
offer_email_report: offer_email_report,
demo: demo,
flags: get_flags(conn.assigns[:current_user]),
is_dbip: is_dbip(),
@ -328,7 +324,6 @@ defmodule PlausibleWeb.StatsController do
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: title(conn, shared_link.site),
offer_email_report: false,
demo: false,
dogfood_page_path: "/share/:dashboard",
shared_link_auth: shared_link.slug,
@ -351,14 +346,6 @@ defmodule PlausibleWeb.StatsController do
end
end
defp remove_email_report_banner(conn, site) do
if conn.assigns[:current_user] do
delete_session(conn, site.domain <> "_offer_email_report")
else
conn
end
end
defp shared_link_cookie_name(slug), do: "shared-link-" <> slug
defp get_flags(_user) do

@ -0,0 +1,228 @@
defmodule PlausibleWeb.Live.CSVImport do
@moduledoc """
LiveView allowing uploading CSVs for imported tables to S3
"""
use PlausibleWeb, :live_view
alias Plausible.Imported.CSVImporter
@impl true
def mount(_params, session, socket) do
%{"site_id" => site_id, "user_id" => user_id} = session
socket =
socket
|> assign(site_id: site_id, user_id: user_id)
|> allow_upload(:import,
accept: [".csv", "text/csv"],
auto_upload: true,
max_entries: length(Plausible.Imported.tables()),
# 1GB
max_file_size: 1_000_000_000,
external: &presign_upload/2,
progress: &handle_progress/3
)
|> process_imported_tables()
{:ok, socket}
end
@impl true
def render(assigns) do
~H"""
<div>
<form action="#" method="post" phx-change="validate-upload-form" phx-submit="submit-upload-form">
<.csv_picker upload={@uploads.import} imported_tables={@imported_tables} />
<.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} />
<p :for={error <- upload_errors(@uploads.import)} class="text-red-400">
<%= error_to_string(error) %>
</p>
</form>
</div>
"""
end
defp csv_picker(assigns) do
~H"""
<label
phx-drop-target={@upload.ref}
class="block border-2 dark:border-gray-600 rounded p-4 group hover:border-indigo-500 dark:hover:border-indigo-600 transition cursor-pointer"
>
<div class="flex items-center">
<div class="bg-gray-200 dark:bg-gray-600 rounded p-1 group-hover:bg-indigo-500 dark:group-hover:bg-indigo-600 transition">
<Heroicons.document_plus class="w-5 h-5 group-hover:text-white transition" />
</div>
<span class="ml-2 text-sm text-gray-600 dark:text-gray-500">
(or drag-and-drop your unzipped CSVs here)
</span>
<.live_file_input upload={@upload} class="hidden" />
</div>
<ul id="imported-tables" class="mt-3.5 mb-0.5 space-y-1.5">
<.imported_table
:for={{table, upload} <- @imported_tables}
table={table}
upload={upload}
errors={if(upload, do: upload_errors(@upload, upload), else: [])}
/>
</ul>
</label>
"""
end
defp confirm_button(assigns) do
~H"""
<button
type="submit"
disabled={not @can_confirm?}
class={[
"rounded-md w-full bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-gray-400 dark:disabled:bg-gray-700 mt-4",
unless(@can_confirm?, do: "cursor-not-allowed")
]}
>
<%= if @date_range do %>
Confirm import from <%= @date_range.first %> to <%= @date_range.last %>
<% else %>
Confirm import
<% end %>
</button>
"""
end
defp imported_table(assigns) do
status =
cond do
assigns.upload && assigns.upload.progress == 100 -> :success
assigns.upload && assigns.upload.progress > 0 -> :in_progress
not Enum.empty?(assigns.errors) -> :error
true -> :empty
end
assigns = assign(assigns, status: status)
~H"""
<li id={@table} class="ml-1.5">
<div class="flex items-center space-x-2">
<Heroicons.document_check :if={@status == :success} class="w-4 h-4 text-indigo-600" />
<PlausibleWeb.Components.Generic.spinner
:if={@status == :in_progress}
class="w-4 h-4 text-indigo-600"
/>
<Heroicons.document :if={@status == :empty} class="w-4 h-4 text-gray-400 dark:text-gray-500" />
<Heroicons.document :if={@status == :error} class="w-4 h-4 text-red-600 dark:text-red-700" />
<span class={[
"text-sm",
if(@upload, do: "dark:text-gray-400", else: "text-gray-400 dark:text-gray-500"),
if(@status == :error, do: "text-red-600 dark:text-red-700")
]}>
<%= if @upload do %>
<%= @upload.client_name %>
<% else %>
<%= @table %>_YYYYMMDD_YYYYMMDD.csv
<% end %>
</span>
</div>
<p :for={error <- @errors} class="ml-6 text-sm text-red-600 dark:text-red-700">
<%= error_to_string(error) %>
</p>
</li>
"""
end
@impl true
def handle_event("validate-upload-form", _params, socket) do
{:noreply, process_imported_tables(socket)}
end
def handle_event("submit-upload-form", _params, socket) do
%{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns
site = Plausible.Repo.get!(Plausible.Site, site_id)
user = Plausible.Repo.get!(Plausible.Auth.User, user_id)
uploads =
consume_uploaded_entries(socket, :import, fn meta, entry ->
{:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}
end)
{:ok, _job} =
CSVImporter.new_import(site, user,
start_date: date_range.first,
end_date: date_range.last,
uploads: uploads
)
redirect_to =
Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain))
{:noreply, redirect(socket, external: redirect_to)}
end
defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"
defp error_to_string(:too_many_files), do: "too many files"
defp error_to_string(:not_accepted), do: "unacceptable file types"
defp error_to_string(:external_client_failure), do: "browser upload failed"
defp presign_upload(entry, socket) do
%{s3_url: s3_url, presigned_url: upload_url} =
Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name)
{:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}
end
defp handle_progress(:import, entry, socket) do
if entry.done? do
{:noreply, process_imported_tables(socket)}
else
{:noreply, socket}
end
end
defp process_imported_tables(socket) do
tables = Plausible.Imported.tables()
{completed, in_progress} = uploaded_entries(socket, :import)
{valid_uploads, invalid_uploads} =
Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))
imported_tables_all_uploads =
Enum.map(tables, fn table ->
uploads =
Enum.filter(valid_uploads, fn upload ->
CSVImporter.extract_table(upload.client_name) == table
end)
{upload, replaced_uploads} = List.pop_at(uploads, -1)
{table, upload, replaced_uploads}
end)
imported_tables =
Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->
{table, upload}
end)
replaced_uploads =
Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->
replaced_uploads
end)
date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))
all_uploaded? = completed != [] and in_progress == []
socket
|> cancel_uploads(invalid_uploads)
|> cancel_uploads(replaced_uploads)
|> assign(
imported_tables: imported_tables,
can_confirm?: all_uploaded?,
date_range: date_range
)
end
defp cancel_uploads(socket, uploads) do
Enum.reduce(uploads, socket, fn upload, socket ->
cancel_upload(socket, :import, upload.ref)
end)
end
end

@ -0,0 +1,304 @@
defmodule PlausibleWeb.Live.Shields.PageRules do
@moduledoc """
LiveView allowing page Rules management
"""
use Phoenix.LiveComponent, global_prefixes: ~w(x-)
use Phoenix.HTML
alias PlausibleWeb.Live.Components.Modal
alias Plausible.Shields
alias Plausible.Shield
import PlausibleWeb.ErrorHelpers
def update(assigns, socket) do
socket =
socket
|> assign(
page_rules_count: assigns[:page_rules_count] || socket.assigns.page_rules_count,
site: assigns[:site] || socket.assigns.site,
current_user: assigns[:current_user] || socket.assigns.current_user,
form: new_form()
)
|> assign_new(:page_rules, fn %{site: site} ->
Shields.list_page_rules(site)
end)
|> assign_new(:redundant_rules, fn %{page_rules: page_rules} ->
detect_redundancy(page_rules)
end)
{:ok, socket}
end
def render(assigns) do
~H"""
<section class="shadow bg-white dark:bg-gray-800 sm:rounded-md sm:overflow-hidden">
<div class="py-6 px-4 sm:p-6">
<header class="relative">
<h2 class="text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">
Pages Block List
</h2>
<p class="mt-1 mb-4 text-sm leading-5 text-gray-500 dark:text-gray-200">
Reject incoming traffic for specific pages
</p>
<PlausibleWeb.Components.Generic.docs_info slug="top-pages#block-traffic-from-specific-pages-or-sections" />
</header>
<div class="border-t border-gray-200 pt-4 grid">
<div
:if={@page_rules_count < Shields.maximum_page_rules()}
class="mt-4 sm:ml-4 sm:mt-0 justify-self-end"
>
<PlausibleWeb.Components.Generic.button
id="add-page-rule"
x-data
x-on:click={Modal.JS.open("page-rule-form-modal")}
>
+ Add Page
</PlausibleWeb.Components.Generic.button>
</div>
<PlausibleWeb.Components.Generic.notice
:if={@page_rules_count >= Shields.maximum_page_rules()}
class="mt-4"
title="Maximum number of pages reached"
>
<p>
You've reached the maximum number of pages you can block (<%= Shields.maximum_page_rules() %>). Please remove one before adding another.
</p>
</PlausibleWeb.Components.Generic.notice>
</div>
<.live_component module={Modal} id="page-rule-form-modal">
<.form
:let={f}
for={@form}
phx-submit="save-page-rule"
phx-target={@myself}
class="max-w-md w-full mx-auto bg-white dark:bg-gray-800"
>
<h2 class="text-xl font-black dark:text-gray-100 mb-8">Add Page to Block List</h2>
<.live_component
submit_name="page_rule[page_path]"
submit_value={f[:page_path].value}
display_value={f[:page_path].value || ""}
module={PlausibleWeb.Live.Components.ComboBox}
suggest_fun={fn input, options -> suggest_page_paths(input, options, @site) end}
id={f[:page_path].id}
creatable
/>
<%= error_tag(f, :page_path) %>
<p class="text-sm mt-2 text-gray-500 dark:text-gray-200">
You can use a wildcard (<code>*</code>) to match multiple pages. For example,
<code>/blog/*</code>
will match <code>/blog/post</code>.
Once added, we will start rejecting traffic from this page within a few minutes.
</p>
<div class="py-4 mt-8">
<PlausibleWeb.Components.Generic.button type="submit" class="w-full">
Add Page →
</PlausibleWeb.Components.Generic.button>
</div>
</.form>
</.live_component>
<p
:if={Enum.empty?(@page_rules)}
class="text-sm text-gray-800 dark:text-gray-200 mt-12 mb-8 text-center"
>
No Page Rules configured for this Site.
</p>
<div
:if={not Enum.empty?(@page_rules)}
class="mt-8 overflow-hidden border-b border-gray-200 shadow dark:border-gray-900 sm:rounded-lg"
>
<table class="min-w-full divide-y divide-gray-200 dark:divide-gray-900">
<thead class="bg-gray-50 dark:bg-gray-900">
<tr>
<th
scope="col"
class="px-6 py-3 text-xs font-medium text-left text-gray-500 uppercase dark:text-gray-100"
>
page
</th>
<th
scope="col"
class="px-6 py-3 text-xs font-medium text-left text-gray-500 uppercase dark:text-gray-100"
>
Status
</th>
<th scope="col" class="px-6 py-3">
<span class="sr-only">Remove</span>
</th>
</tr>
</thead>
<tbody>
<%= for rule <- @page_rules do %>
<tr class="text-gray-900 dark:text-gray-100">
<td class="px-6 py-4 text-sm font-medium max-w-xs truncate text-ellipsis overflow-hidden">
<div class="flex items-center">
<span
id={"page-#{rule.id}"}
class="mr-4 cursor-help border-b border-dotted border-gray-400 text-ellipsis overflow-hidden"
title={"#{rule.page_path}\n\nAdded at #{format_added_at(rule.inserted_at, @site.timezone)} by #{rule.added_by}"}
>
<%= rule.page_path %>
</span>
</div>
</td>
<td class="px-6 py-4 text-sm text-gray-500">
<div class="flex items-center">
<span :if={rule.action == :deny}>
Blocked
</span>
<span :if={rule.action == :allow}>
Allowed
</span>
<span
:if={@redundant_rules[rule.id]}
title={"This rule might be redundant because the following rules may match first:\n\n#{Enum.join(@redundant_rules[rule.id], "\n")}"}
class="pl-4"
>
<Heroicons.exclamation_triangle class="h-4 w-4 text-red-500" />
</span>
</div>
</td>
<td class="px-6 py-4 text-sm font-medium text-right">
<button
id={"remove-page-rule-#{rule.id}"}
phx-target={@myself}
phx-click="remove-page-rule"
phx-value-rule-id={rule.id}
class="text-sm text-red-600"
data-confirm="Are you sure you want to revoke this rule?"
>
Remove
</button>
</td>
</tr>
<% end %>
</tbody>
</table>
</div>
</div>
</section>
"""
end
def handle_event("save-page-rule", %{"page_rule" => params}, socket) do
user = socket.assigns.current_user
case Shields.add_page_rule(
socket.assigns.site.id,
params,
added_by: user
) do
{:ok, rule} ->
page_rules = [rule | socket.assigns.page_rules]
socket =
socket
|> Modal.close("page-rule-form-modal")
|> assign(
form: new_form(),
page_rules: page_rules,
page_rules_count: socket.assigns.page_rules_count + 1,
redundant_rules: detect_redundancy(page_rules)
)
# Make sure to clear the combobox input after adding a page rule, on subsequent modal reopening
send_update(PlausibleWeb.Live.Components.ComboBox,
id: "page_rule_page_code",
display_value: ""
)
send_flash(
:success,
"Page rule added successfully. Traffic will be rejected within a few minutes."
)
{:noreply, socket}
{:error, %Ecto.Changeset{} = changeset} ->
{:noreply, assign(socket, form: to_form(changeset))}
end
end
def handle_event("remove-page-rule", %{"rule-id" => rule_id}, socket) do
Shields.remove_page_rule(socket.assigns.site.id, rule_id)
send_flash(
:success,
"Page rule removed successfully. Traffic will be resumed within a few minutes."
)
page_rules = Enum.reject(socket.assigns.page_rules, &(&1.id == rule_id))
{:noreply,
socket
|> assign(
page_rules_count: socket.assigns.page_rules_count - 1,
page_rules: page_rules,
redundant_rules: detect_redundancy(page_rules)
)}
end
def send_flash(kind, message) do
send(self(), {:flash, kind, message})
end
defp new_form() do
%Shield.PageRule{}
|> Shield.PageRule.changeset(%{})
|> to_form()
end
defp format_added_at(dt, tz) do
dt
|> Plausible.Timezones.to_datetime_in_timezone(tz)
|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end
def suggest_page_paths(input, _options, site) do
query = Plausible.Stats.Query.from(site, %{})
site
|> Plausible.Stats.filter_suggestions(query, "page", input)
|> Enum.map(fn %{label: label, value: value} -> {label, value} end)
end
defp detect_redundancy(page_rules) do
page_rules
|> Enum.reduce(%{}, fn rule, acc ->
{[^rule], remaining_rules} =
Enum.split_with(
page_rules,
fn r -> r == rule end
)
conflicting =
remaining_rules
|> Enum.filter(fn candidate ->
rule
|> Map.fetch!(:page_path_pattern)
|> maybe_compile()
|> Regex.match?(candidate.page_path)
end)
|> Enum.map(& &1.id)
Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc ->
Map.update(acc, conflicting_rule_id, [rule.page_path], fn existing ->
[rule.page_path | existing]
end)
end)
end)
end
defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern)
defp maybe_compile(%Regex{} = pattern), do: pattern
end

@ -0,0 +1,53 @@
defmodule PlausibleWeb.Live.Shields.Pages do
@moduledoc """
LiveView for IP Addresses Shield
"""
use PlausibleWeb, :live_view
use Phoenix.HTML
alias Plausible.Shields
alias Plausible.Sites
def mount(
_params,
%{
"domain" => domain,
"current_user_id" => user_id
},
socket
) do
socket =
socket
|> assign_new(:site, fn ->
Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin])
end)
|> assign_new(:page_rules_count, fn %{site: site} ->
Shields.count_page_rules(site)
end)
|> assign_new(:current_user, fn ->
Plausible.Repo.get(Plausible.Auth.User, user_id)
end)
{:ok, socket}
end
def render(assigns) do
~H"""
<div>
<.flash_messages flash={@flash} />
<.live_component
module={PlausibleWeb.Live.Shields.PageRules}
current_user={@current_user}
page_rules_count={@page_rules_count}
site={@site}
id="page-rules-#{@current_user.id}"
/>
</div>
"""
end
def handle_info({:flash, kind, message}, socket) do
socket = put_live_flash(socket, kind, message)
{:noreply, socket}
end
end

@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router do
delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
post "/:website/settings/export", SiteController, :export
post "/:website/settings/export", SiteController, :csv_export
get "/:website/settings/import", SiteController, :csv_import
get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats

@ -0,0 +1,20 @@
<div class="max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded p-6 mb-4 mt-8">
<h2 class="text-xl font-black dark:text-gray-100">Import from CSV files</h2>
<div class="my-3 space-y-1.5 text-sm text-gray-400">
<p>
Please ensure each file follows
<.styled_link href="https://plausible.io/docs/csv-import">
our CSV format guidelines.
</.styled_link>
</p>
<p>
You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area.
</p>
</div>
<%= live_render(@conn, PlausibleWeb.Live.CSVImport,
session: %{"site_id" => @site.id, "user_id" => @current_user.id}
) %>
</div>

@ -20,9 +20,9 @@
</PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20 opacity-40 cursor-not-allowed"
class="w-36 h-20"
theme="bright"
href=""
href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</PlausibleWeb.Components.Generic.button_link>

@ -14,4 +14,11 @@
"domain" => @site.domain
}
) %>
<% "pages" -> %>
<%= live_render(@conn, PlausibleWeb.Live.Shields.Pages,
session: %{
"site_id" => @site.id,
"domain" => @site.domain
}
) %>
<% end %>

@ -1,3 +1,8 @@
<PlausibleWeb.Components.FirstDashboardLaunchBanner.set
:if={@conn.params["site_created"] == "true"}
site={@site}
/>
<div class="w-full max-w-3xl mt-4 mx-auto flex">
<%= form_for @conn, "/", [class: "max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-bold dark:text-gray-100">Add JavaScript snippet</h2>

@ -1,63 +0,0 @@
<div class="<%= stats_container_class(@conn) %>" data-site-domain="<%= @site.domain %>">
<%= if @offer_email_report do %>
<div class="w-full px-4 text-sm font-bold text-center text-blue-900 bg-blue-200 rounded transition" style="top: 91px" role="alert">
<%= link("Team members, email reports and GA import. Explore more →", to: "/#{URI.encode_www_form(@site.domain)}/settings/email-reports", class: "py-2 block") %>
</div>
<% end %>
<%= if @site.locked do %>
<div class="w-full px-4 py-4 text-sm font-bold text-center text-yellow-800 bg-yellow-100 rounded transition" style="top: 91px" role="alert">
<p>This dashboard is actually locked. You are viewing it with super-admin access</p>
</div>
<% end %>
<div class="pt-6"></div>
<div
id="stats-react-container"
style="overflow-anchor: none;"
data-domain="<%= @site.domain %>"
data-offset="<%= Plausible.Site.tz_offset(@site) %>"
data-has-goals="<%= @has_goals %>"
data-conversions-opted-out="<%= Plausible.Billing.Feature.Goals.opted_out?(@site) %>"
data-funnels-opted-out="<%= Plausible.Billing.Feature.Funnels.opted_out?(@site) %>"
data-props-opted-out="<%= Plausible.Billing.Feature.Props.opted_out?(@site) %>"
data-funnels-available="<%= Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok %>"
data-props-available="<%= Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok %>"
data-funnels="<%= Jason.encode!(@funnels) %>"
data-has-props="<%= @has_props %>"
data-logged-in="<%= !!@conn.assigns[:current_user] %>"
data-stats-begin="<%= @stats_start_date %>"
data-native-stats-begin="<%= @native_stats_start_date %>"
data-shared-link-auth="<%= assigns[:shared_link_auth] %>"
data-embedded="<%= @conn.assigns[:embedded] %>"
data-background="<%= @conn.assigns[:background] %>"
data-is-dbip="<%= @is_dbip %>"
data-current-user-role="<%= @conn.assigns[:current_user_role] %>"
data-flags="<%= Jason.encode!(@flags) %>"
data-valid-intervals-by-period="<%= Plausible.Stats.Interval.valid_by_period(site: @site) |> Jason.encode!() %>">
</div>
<div id="modal_root"></div>
<%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
<div class="bg-gray-50 dark:bg-gray-850">
<div class="py-12 lg:py-16 lg:flex lg:items-center lg:justify-between">
<h2 class="text-3xl font-extrabold tracking-tight text-gray-900 leading-9 sm:text-4xl sm:leading-10 dark:text-gray-100">
Want these stats for your website?
<br />
<span class="text-indigo-600">Start your free trial today.</span>
</h2>
<div class="flex mt-8 lg:flex-shrink-0 lg:mt-0">
<div class="inline-flex shadow rounded-md">
<a href="/register" class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-white bg-indigo-600 border border-transparent leading-6 rounded-md hover:bg-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out">
Get started
</a>
</div>
<div class="inline-flex ml-3 shadow rounded-md">
<a href="/" class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-indigo-600 bg-white border border-transparent leading-6 rounded-md dark:text-gray-100 dark:bg-gray-800 hover:text-indigo-500 dark:hover:text-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out">
Learn more
</a>
</div>
</div>
</div>
</div>
<% end %>
</div>

@ -0,0 +1,75 @@
<div class={stats_container_class(@conn)} data-site-domain={@site.domain}>
<PlausibleWeb.Components.FirstDashboardLaunchBanner.render site={@site} />
<%= if @site.locked do %>
<div
class="w-full px-4 py-4 text-sm font-bold text-center text-yellow-800 bg-yellow-100 rounded transition"
style="top: 91px"
role="alert"
>
<p>This dashboard is actually locked. You are viewing it with super-admin access</p>
</div>
<% end %>
<div class="pt-6"></div>
<div
id="stats-react-container"
style="overflow-anchor: none;"
data-domain={@site.domain}
data-offset={Plausible.Site.tz_offset(@site)}
data-has-goals={to_string(@has_goals)}
data-conversions-opted-out={to_string(Plausible.Billing.Feature.Goals.opted_out?(@site))}
data-funnels-opted-out={to_string(Plausible.Billing.Feature.Funnels.opted_out?(@site))}
data-props-opted-out={to_string(Plausible.Billing.Feature.Props.opted_out?(@site))}
data-funnels-available={
to_string(Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok)
}
data-props-available={
to_string(Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok)
}
data-funnels={Jason.encode!(@funnels)}
data-has-props={to_string(@has_props)}
data-logged-in={to_string(!!@conn.assigns[:current_user])}
data-stats-begin={@stats_start_date}
data-native-stats-begin={@native_stats_start_date}
data-shared-link-auth={assigns[:shared_link_auth]}
data-embedded={to_string(@conn.assigns[:embedded])}
data-background={@conn.assigns[:background]}
data-is-dbip={to_string(@is_dbip)}
data-current-user-role={@conn.assigns[:current_user_role]}
data-flags={Jason.encode!(@flags)}
data-valid-intervals-by-period={
Plausible.Stats.Interval.valid_by_period(site: @site) |> Jason.encode!()
}
>
</div>
<div id="modal_root"></div>
<%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
<div class="bg-gray-50 dark:bg-gray-850">
<div class="py-12 lg:py-16 lg:flex lg:items-center lg:justify-between">
<h2 class="text-3xl font-extrabold tracking-tight text-gray-900 leading-9 sm:text-4xl sm:leading-10 dark:text-gray-100">
Want these stats for your website? <br />
<span class="text-indigo-600">Start your free trial today.</span>
</h2>
<div class="flex mt-8 lg:flex-shrink-0 lg:mt-0">
<div class="inline-flex shadow rounded-md">
<a
href="/register"
class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-white bg-indigo-600 border border-transparent leading-6 rounded-md hover:bg-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out"
>
Get started
</a>
</div>
<div class="inline-flex ml-3 shadow rounded-md">
<a
href="/"
class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-indigo-600 bg-white border border-transparent leading-6 rounded-md dark:text-gray-100 dark:bg-gray-800 hover:text-indigo-500 dark:hover:text-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out"
>
Learn more
</a>
</div>
</div>
</div>
</div>
<% end %>
</div>

@ -65,10 +65,15 @@ defmodule PlausibleWeb.LayoutView do
%{
key: "Shields",
icon: :shield_exclamation,
value: [
%{key: "IP Addresses", value: "shields/ip_addresses"},
%{key: "Countries", value: "shields/countries"}
]
value:
[
%{key: "IP Addresses", value: "shields/ip_addresses"},
%{key: "Countries", value: "shields/countries"},
if FunWithFlags.enabled?(:shield_pages, for: conn.assigns[:site]) do
%{key: "Pages", value: "shields/pages"}
end
]
|> Enum.reject(&is_nil/1)
},
%{key: "Email Reports", value: "email-reports", icon: :envelope},
if conn.assigns[:current_user_role] == :owner do

@ -0,0 +1,17 @@
defmodule Plausible.Repo.Migrations.ShieldPageRules do
use Ecto.Migration
def change do
create table(:shield_rules_page, primary_key: false) do
add(:id, :uuid, primary_key: true)
add :site_id, references(:sites, on_delete: :delete_all), null: false
add :page_path, :text, null: false
add :page_path_pattern, :text, null: false
add :action, :string, default: "deny", null: false
add :added_by, :string
timestamps()
end
create unique_index(:shield_rules_page, [:site_id, :page_path_pattern])
end
end

@ -11,6 +11,7 @@
# and so on) as they will fail if something goes wrong.
FunWithFlags.enable(:imports_exports)
FunWithFlags.enable(:shield_pages)
user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible")

@ -158,6 +158,25 @@ defmodule Plausible.Ingestion.EventTest do
assert dropped.drop_reason == :site_country_blocklist
end
test "event pipeline drops a request when page is on blocklist" do
site = insert(:site)
payload = %{
name: "pageview",
url: "http://dummy.site/blocked/page",
domain: site.domain
}
conn = build_conn(:post, "/api/events", payload)
{:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/blocked/**"})
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
assert dropped.drop_reason == :site_page_blocklist
end
test "event pipeline drops events for site with accept_trafic_until in the past" do
yesterday = Date.add(Date.utc_today(), -1)

@ -0,0 +1,107 @@
defmodule Plausible.Shield.PageRuleCacheTest do
use Plausible.DataCase, async: true
alias Plausible.Shield.PageRule
alias Plausible.Shield.PageRuleCache
alias Plausible.Shields
describe "public cache interface" do
test "cache caches page rules", %{test: test} do
cache_opts = [force?: true, cache_name: test]
{:ok, _} =
Supervisor.start_link(
[
{PageRuleCache,
[cache_name: test, child_id: :page_rules_cache_id, ets_options: [:bag]]}
],
strategy: :one_for_one,
name: :"cache_supervisor_#{test}"
)
site1 = insert(:site)
site2 = insert(:site)
{:ok, %{id: rid1}} = Shields.add_page_rule(site1, %{"page_path" => "/test/1"})
{:ok, %{id: rid2}} = Shields.add_page_rule(site1, %{"page_path" => "/test/2"})
{:ok, %{id: rid3}} = Shields.add_page_rule(site2, %{"page_path" => "/test/2"})
:ok = PageRuleCache.refresh_all(cache_name: test)
:ok = Shields.remove_page_rule(site1, rid1)
assert PageRuleCache.size(test) == 3
# the rule order should be deterministic, but with 1s timestamp sorting precision
# race conditions may happen during tests
assert rules = PageRuleCache.get(site1.domain, cache_opts)
rule_ids = Enum.map(rules, & &1.id)
assert rid1 in rule_ids
assert rid2 in rule_ids
assert length(rules) == 2
assert %PageRule{from_cache?: true, id: ^rid3} =
PageRuleCache.get(site2.domain, cache_opts)
refute PageRuleCache.get("rogue.example.com", cache_opts)
end
test "page path patterns are already compiled when fetched from cache", %{test: test} do
site = insert(:site)
{:ok, _} = start_test_cache(test)
cache_opts = [force?: true, cache_name: test]
{:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/hello/**/world"})
:ok = PageRuleCache.refresh_all(cache_name: test)
assert regex = PageRuleCache.get(site.domain, cache_opts).page_path_pattern
assert regex == ~r/^\/hello\/.*\/world$/
end
test "cache allows lookups for page paths on sites with changed domain", %{test: test} do
{:ok, _} = start_test_cache(test)
cache_opts = [force?: true, cache_name: test]
site = insert(:site, domain: "new.example.com", domain_changed_from: "old.example.com")
{:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/#{test}"})
:ok = PageRuleCache.refresh_all(cache_name: test)
assert PageRuleCache.get("old.example.com", cache_opts)
assert PageRuleCache.get("new.example.com", cache_opts)
refute PageRuleCache.get("rogue.example.com", cache_opts)
end
test "refreshes only recently added pages rules", %{test: test} do
{:ok, _} = start_test_cache(test)
domain = "site1.example.com"
site = insert(:site, domain: domain)
cache_opts = [cache_name: test, force?: true]
yesterday =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(-1 * 60 * 60 * 24)
|> NaiveDateTime.truncate(:second)
{:ok, r1} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/1"})
Ecto.Changeset.change(r1, inserted_at: yesterday, updated_at: yesterday) |> Repo.update!()
{:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/2"})
assert PageRuleCache.get(domain, cache_opts) == nil
assert :ok = PageRuleCache.refresh_updated_recently(cache_opts)
assert %{page_path_pattern: ~r[^/test/2$]} = PageRuleCache.get(domain, cache_opts)
assert :ok = PageRuleCache.refresh_all(cache_opts)
assert [_, _] = PageRuleCache.get(domain, cache_opts)
end
end
defp start_test_cache(cache_name) do
%{start: {m, f, a}} = PageRuleCache.child_spec(cache_name: cache_name, ets_options: [:bag])
apply(m, f, a)
end
end

@ -1,270 +0,0 @@
defmodule Plausible.ShieldsTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_ip_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{})
assert changeset.errors == [inet: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported netmask", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"})
assert changeset.errors == [inet: {"netmask unsupported", []}]
refute changeset.valid?
assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"})
end
test "incorrect ip", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"})
assert changeset.errors == [
inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]}
]
refute changeset.valid?
end
test "non-strict IPs", %{site: site} do
assert {:error, _} = add_ip_rule(site, %{"inet" => "111"})
end
test "double insert", %{site: site} do
assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
refute changeset.valid?
assert changeset.errors == [
inet:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_ip_site_id_inet_index"}
]}
]
end
test "ipv6", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "ipv4", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "over limit", %{site: site} do
for i <- 1..maximum_ip_rules() do
assert {:ok, _} =
add_ip_rule(site, %{"inet" => "1.1.1.#{i}"})
end
assert count_ip_rules(site) == maximum_ip_rules()
assert {:error, changeset} =
add_ip_rule(site, %{"inet" => "1.1.1.31"})
refute changeset.valid?
assert changeset.errors == [inet: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_ip_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert remove_ip_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.IPRule, rule.id)
assert remove_ip_rule(site, rule.id) == :ok
end
end
describe "list_ip_rules/1" do
test "empty", %{site: site} do
assert(list_ip_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
:timer.sleep(1000)
{:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert [^r2, ^r1] = list_ip_rules(site)
end
end
describe "count_ip_rules/1" do
test "counts", %{site: site} do
assert count_ip_rules(site) == 0
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert count_ip_rules(site) == 1
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert count_ip_rules(site) == 2
end
end
describe "IP Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_ip_rules(site.id) == 0
assert list_ip_rules(site.id) == []
assert {:ok, rule} =
add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"})
add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"})
assert count_ip_rules(site) == 1
assert [^rule] = list_ip_rules(site)
assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32}
assert rule.description == "Localhost"
assert rule.action == :deny
refute rule.from_cache?
end
end
describe "add_country_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{})
assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported country", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"})
assert changeset.errors == [country_code: {"is invalid", []}]
refute changeset.valid?
end
test "incorrect country format", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"})
assert changeset.errors ==
[
{:country_code, {"is invalid", []}},
{:country_code,
{"should be %{count} character(s)",
[count: 2, validation: :length, kind: :is, type: :string]}}
]
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"})
refute changeset.valid?
assert changeset.errors == [
country_code:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_country_site_id_country_code_index"}
]}
]
end
test "over limit", %{site: site} do
country_codes =
Location.Country.all()
|> Enum.take(Plausible.Shields.maximum_country_rules())
|> Enum.map(& &1.alpha_2)
for cc <- country_codes do
assert {:ok, _} =
add_country_rule(site, %{"country_code" => cc})
end
assert count_country_rules(site) == maximum_country_rules()
assert {:error, changeset} =
add_country_rule(site, %{"country_code" => "US"})
refute changeset.valid?
assert changeset.errors == [country_code: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_country_rule(site, %{"country_code" => "EE"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_country_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_country_rule(site, %{"country_code" => "EE"})
assert remove_country_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.CountryRule, rule.id)
assert remove_country_rule(site, rule.id) == :ok
end
end
describe "list_country_rules/1" do
test "empty", %{site: site} do
assert(list_country_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_country_rule(site, %{"country_code" => "EE"})
:timer.sleep(1000)
{:ok, r2} = add_country_rule(site, %{"country_code" => "PL"})
assert [^r2, ^r1] = list_country_rules(site)
end
end
describe "count_country_rules/1" do
test "counts", %{site: site} do
assert count_country_rules(site) == 0
{:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert count_country_rules(site) == 1
{:ok, _} = add_country_rule(site, %{"country_code" => "PL"})
assert count_country_rules(site) == 2
end
end
describe "Country Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_country_rules(site.id) == 0
assert list_country_rules(site.id) == []
assert {:ok, rule} =
add_country_rule(site.id, %{"country_code" => "EE"})
add_country_rule(site2, %{"country_code" => "PL"})
assert count_country_rules(site) == 1
assert [^rule] = list_country_rules(site)
assert rule.country_code == "EE"
assert rule.action == :deny
refute rule.from_cache?
end
end
end

@ -1,2 +1,2 @@
name,conversions,conversion_rate
,1,100.0
/signup,1,100.0

1 name conversions conversion_rate
2 /signup 1 100.0

@ -1,2 +1,2 @@
name,conversions,conversion_rate
,1,100.0
/signup,1,100.0

1 name conversions conversion_rate
2 /signup 1 100.0

@ -1,3 +1,3 @@
property,value,visitors,events,percentage
variant,(none),3,4,75.0
variant,A,1,1,25.0
variant,A,1,2,25.0

1 property value visitors events percentage
2 variant (none) 3 4 75.0
3 variant A 1 1 2 25.0

@ -1,3 +1,3 @@
name,unique_entrances,total_entrances,visit_duration
/,3,3,20
,1,1,0
/signup,1,1,60

1 name unique_entrances total_entrances visit_duration
2 / 3 3 20
3 /signup 1 1 0 60

@ -1,4 +1,4 @@
name,unique_exits,total_exits,exit_rate
/,2,2,66.0
,1,1,
/signup,1,1,100.0
/some-other-page,1,1,100.0

1 name unique_exits total_exits exit_rate
2 / 2 2 66.0
3 /signup 1 1 100.0
4 /some-other-page 1 1 100.0

@ -1,3 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page
/,4,3,67,
/signup,1,1,0,60.0
/some-other-page,1,1,,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 4 3 67
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 60.0

@ -1,2 +1,2 @@
name,visitors,bounce_rate,visit_duration
Direct / None,4,75,15
Direct / None,4,50,30

1 name visitors bounce_rate visit_duration
2 Direct / None 4 75 50 15 30

@ -1,3 +1,3 @@
name,visitors,bounce_rate,visit_duration
Direct / None,3,100,0
Direct / None,3,67,20
Google,1,0,60

1 name visitors bounce_rate visit_duration
2 Direct / None 3 100 67 0 20
3 Google 1 0 60

@ -28,5 +28,5 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration
2021-10-16,0,0,0,0.0,,
2021-10-17,0,0,0,0.0,,
2021-10-18,0,0,0,0.0,,
2021-10-19,2,1,2,0.5,100,0
2021-10-19,2,2,2,1.0,50,30
2021-10-20,1,2,1,2.0,0,60

1 date visitors pageviews visits views_per_visit bounce_rate visit_duration
28 2021-10-16 0 0 0 0.0
29 2021-10-17 0 0 0 0.0
30 2021-10-18 0 0 0 0.0
31 2021-10-19 2 1 2 2 0.5 1.0 100 50 0 30
32 2021-10-20 1 2 1 2.0 0 60

@ -1,3 +1,3 @@
property,value,visitors,events,percentage
variant,(none),4,5,80.0
variant,A,1,1,20.0
variant,A,1,2,20.0

1 property value visitors events percentage
2 variant (none) 4 5 80.0
3 variant A 1 1 2 20.0

@ -1,3 +1,3 @@
name,unique_entrances,total_entrances,visit_duration
/,4,4,15
,1,1,0
/signup,1,1,60

1 name unique_entrances total_entrances visit_duration
2 / 4 4 15
3 /signup 1 1 0 60

@ -1,4 +1,4 @@
name,unique_exits,total_exits,exit_rate
/,3,3,75.0
,1,1,
/signup,1,1,100.0
/some-other-page,1,1,100.0

1 name unique_exits total_exits exit_rate
2 / 3 3 75.0
3 /signup 1 1 100.0
4 /some-other-page 1 1 100.0

@ -1,3 +1,4 @@
name,visitors,pageviews,bounce_rate,time_on_page
/,5,4,75,
/signup,1,1,0,60.0
/some-other-page,1,1,,60.0

1 name visitors pageviews bounce_rate time_on_page
2 / 5 4 75
3 /signup 1 1 0 60.0
4 /some-other-page 1 1 60.0

@ -1,2 +1,2 @@
name,visitors,bounce_rate,visit_duration
Direct / None,5,80,12
Direct / None,5,60,24

1 name visitors bounce_rate visit_duration
2 Direct / None 5 80 60 12 24

@ -1,3 +1,3 @@
name,visitors,bounce_rate,visit_duration
Direct / None,3,100,0
Direct / None,3,67,20
Google,2,50,30

1 name visitors bounce_rate visit_duration
2 Direct / None 3 100 67 0 20
3 Google 2 50 30

@ -4,4 +4,4 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration
2021-07-01,0,0,0,0.0,,
2021-08-01,0,0,0,0.0,,
2021-09-01,1,1,1,1.0,100,0
2021-10-01,3,3,3,1.0,67,20
2021-10-01,3,4,3,1.33,33,40

1 date visitors pageviews visits views_per_visit bounce_rate visit_duration
4 2021-07-01 0 0 0 0.0
5 2021-08-01 0 0 0 0.0
6 2021-09-01 1 1 1 1.0 100 0
7 2021-10-01 3 3 4 3 1.0 1.33 67 33 20 40

@ -1324,6 +1324,24 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
}
]
end
test "ignores entry pages from sessions with only custom events", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
name: "Signup",
timestamp: ~N[2021-01-01 00:15:00],
pathname: "/"
)
])
conn =
get(
conn,
"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01"
)
assert json_response(conn, 200) == []
end
end
describe "GET /api/stats/:domain/exit-pages" do
@ -1557,5 +1575,23 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
%{"name" => "/exit2", "visitors" => 1, "visits" => 1}
]
end
test "ignores exit pages from sessions with only custom events", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,
name: "Signup",
timestamp: ~N[2021-01-01 00:15:00],
pathname: "/"
)
])
conn =
get(
conn,
"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01"
)
assert json_response(conn, 200) == []
end
end
end

@ -244,7 +244,9 @@ defmodule PlausibleWeb.SiteControllerTest do
}
})
assert redirected_to(conn) == "/#{URI.encode_www_form("éxample.com")}/snippet"
assert redirected_to(conn) ==
"/#{URI.encode_www_form("éxample.com")}/snippet?site_created=true"
assert site = Repo.get_by(Plausible.Site, domain: "éxample.com")
assert site.timezone == "Europe/London"
assert site.ingest_rate_limit_scale_seconds == 60
@ -341,7 +343,7 @@ defmodule PlausibleWeb.SiteControllerTest do
}
})
assert redirected_to(conn) == "/example.com/snippet"
assert redirected_to(conn) == "/example.com/snippet?site_created=true"
assert Repo.get_by(Plausible.Site, domain: "example.com")
end
@ -361,7 +363,7 @@ defmodule PlausibleWeb.SiteControllerTest do
}
})
assert redirected_to(conn) == "/example.com/snippet"
assert redirected_to(conn) == "/example.com/snippet?site_created=true"
assert Plausible.Billing.Quota.site_usage(user) == 3
end
@ -375,7 +377,7 @@ defmodule PlausibleWeb.SiteControllerTest do
}
})
assert redirected_to(conn) == "/example.com/snippet"
assert redirected_to(conn) == "/example.com/snippet?site_created=true"
assert Repo.get_by(Plausible.Site, domain: "example.com")
end
end

@ -16,6 +16,18 @@ defmodule PlausibleWeb.StatsControllerTest do
assert text_of_attr(resp, @react_container, "data-domain") == site.domain
assert text_of_attr(resp, @react_container, "data-is-dbip") == "false"
assert text_of_attr(resp, @react_container, "data-has-goals") == "false"
assert text_of_attr(resp, @react_container, "data-conversions-opted-out") == "false"
assert text_of_attr(resp, @react_container, "data-funnels-opted-out") == "false"
assert text_of_attr(resp, @react_container, "data-props-opted-out") == "false"
assert text_of_attr(resp, @react_container, "data-props-available") == "true"
assert text_of_attr(resp, @react_container, "data-funnels-available") == "true"
assert text_of_attr(resp, @react_container, "data-has-props") == "false"
assert text_of_attr(resp, @react_container, "data-logged-in") == "false"
assert text_of_attr(resp, @react_container, "data-embedded") == ""
[{"div", attrs, _}] = find(resp, @react_container)
assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end)
assert ["noindex, nofollow"] ==
resp
@ -61,7 +73,8 @@ defmodule PlausibleWeb.StatsControllerTest do
test "can view stats of a website I've created", %{conn: conn, site: site} do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/" <> site.domain)
assert html_response(conn, 200) =~ "stats-react-container"
resp = html_response(conn, 200)
assert text_of_attr(resp, @react_container, "data-logged-in") == "true"
end
test "shows locked page if page is locked", %{conn: conn, user: user} do
@ -102,8 +115,11 @@ defmodule PlausibleWeb.StatsControllerTest do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/" <> site.domain)
assert html_response(conn, 200) =~ "stats-react-container"
assert html_response(conn, 200) =~ "This dashboard is actually locked"
resp = html_response(conn, 200)
assert resp =~ "This dashboard is actually locked"
[{"div", attrs, _}] = find(resp, @react_container)
assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end)
end
test "can view a private locked dashboard without stats", %{conn: conn} do
@ -120,7 +136,10 @@ defmodule PlausibleWeb.StatsControllerTest do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/" <> site.domain)
assert html_response(conn, 200) =~ "stats-react-container"
resp = html_response(conn, 200)
[{"div", attrs, _}] = find(resp, @react_container)
assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end)
end
end
@ -285,7 +304,7 @@ defmodule PlausibleWeb.StatsControllerTest do
["2021-09-27", "0", "0", "0", "0.0", "", ""],
["2021-10-04", "0", "0", "0", "0.0", "", ""],
["2021-10-11", "0", "0", "0", "0.0", "", ""],
["2021-10-18", "3", "3", "3", "1.0", "67", "20"],
["2021-10-18", "3", "4", "3", "1.33", "33", "40"],
[""]
]
end
@ -469,7 +488,17 @@ defmodule PlausibleWeb.StatsControllerTest do
browser: "FirefoxNoVersion",
operating_system: "MacNoVersion"
),
build(:pageview,
user_id: 456,
timestamp:
Timex.shift(~N[2021-10-20 12:00:00], days: -1, minutes: -1)
|> NaiveDateTime.truncate(:second),
pathname: "/signup",
"meta.key": ["variant"],
"meta.value": ["A"]
),
build(:event,
user_id: 456,
timestamp:
Timex.shift(~N[2021-10-20 12:00:00], days: -1) |> NaiveDateTime.truncate(:second),
name: "Signup",
@ -606,7 +635,24 @@ defmodule PlausibleWeb.StatsControllerTest do
link = insert(:shared_link, site: site)
conn = get(conn, "/share/test-site.com/?auth=#{link.slug}")
resp = html_response(conn, 200)
assert text_of_attr(resp, @react_container, "data-embedded") == "false"
assert Plug.Conn.get_resp_header(conn, "x-frame-options") == []
end
test "returns page embedded page", %{
conn: conn
} do
site = insert(:site, domain: "test-site.com")
link = insert(:shared_link, site: site)
conn = get(conn, "/share/test-site.com/?auth=#{link.slug}&embed=true")
resp = html_response(conn, 200)
assert text_of_attr(resp, @react_container, "data-embedded") == "true"
assert Plug.Conn.get_resp_header(conn, "x-frame-options") == []
[{"div", attrs, _}] = find(resp, @react_container)
assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end)
end
test "shows locked page if page is locked", %{conn: conn} do

@ -95,7 +95,7 @@ defmodule PlausibleWeb.Live.Shields.IPAddressesTest do
assert element_exists?(html, ~s/a[phx-click="prefill-own-ip-rule"]/)
end
test "form modal does not contains link to add own IP if already added", %{
test "form modal does not contain link to add own IP if already added", %{
site: site,
conn: conn
} do

@ -0,0 +1,176 @@
defmodule PlausibleWeb.Live.Shields.PagesTest do
use PlausibleWeb.ConnCase, async: true
import Phoenix.LiveViewTest
import Plausible.Test.Support.HTML
alias Plausible.Shields
setup [:create_user, :create_site, :log_in]
describe "Page Rules - static" do
test "renders page rules page with empty list", %{conn: conn, site: site} do
conn = get(conn, "/#{site.domain}/settings/shields/pages")
resp = html_response(conn, 200)
assert resp =~ "No Page Rules configured for this Site"
assert resp =~ "Pages Block List"
end
test "lists page rules with remove actions", %{conn: conn, site: site} do
{:ok, r1} =
Shields.add_page_rule(site, %{"page_path" => "/test/1"})
{:ok, r2} =
Shields.add_page_rule(site, %{"page_path" => "/test/2"})
conn = get(conn, "/#{site.domain}/settings/shields/pages")
resp = html_response(conn, 200)
assert resp =~ "/test/1"
assert resp =~ "/test/2"
assert remove_button_1 = find(resp, "#remove-page-rule-#{r1.id}")
assert remove_button_2 = find(resp, "#remove-page-rule-#{r2.id}")
assert text_of_attr(remove_button_1, "phx-click" == "remove-page-rule")
assert text_of_attr(remove_button_1, "phx-value-rule-id" == r1.id)
assert text_of_attr(remove_button_2, "phx-click" == "remove-page-rule")
assert text_of_attr(remove_button_2, "phx-value-rule-id" == r2.id)
end
test "add rule button is rendered", %{conn: conn, site: site} do
conn = get(conn, "/#{site.domain}/settings/shields/pages")
resp = html_response(conn, 200)
assert element_exists?(resp, ~s/button#add-page-rule[x-data]/)
attr = text_of_attr(resp, ~s/button#add-page-rule/, "x-on:click")
assert attr =~ "open-modal"
assert attr =~ "page-rule-form-modal"
end
test "add rule button is not rendered when maximum reached", %{conn: conn, site: site} do
for i <- 1..Shields.maximum_page_rules() do
assert {:ok, _} =
Shields.add_page_rule(site, %{"page_path" => "/test/#{i}"})
end
conn = get(conn, "/#{site.domain}/settings/shields/pages")
resp = html_response(conn, 200)
refute element_exists?(resp, ~s/button#add-page-rule[x-data]/)
assert resp =~ "Maximum number of pages reached"
assert resp =~ "You've reached the maximum number of pages you can block (30)"
end
end
describe "Page Rules - LiveView" do
test "modal contains form", %{site: site, conn: conn} do
lv = get_liveview(conn, site)
html = render(lv)
assert element_exists?(
html,
~s/form[phx-submit="save-page-rule"] input[name="page_rule\[page_path\]"]/
)
assert submit_button(html, ~s/form[phx-submit="save-page-rule"]/)
end
test "submitting a valid Page saves it", %{conn: conn, site: site} do
lv = get_liveview(conn, site)
lv
|> element("form")
|> render_submit(%{
"page_rule[page_path]" => "/test/**"
})
html = render(lv)
assert html =~ "/test/**"
assert [%{page_path: "/test/**"}] = Shields.list_page_rules(site)
end
test "submitting invalid Page renders error", %{conn: conn, site: site} do
lv = get_liveview(conn, site)
lv
|> element("form")
|> render_submit(%{
"page_rule[page_path]" => "WRONG"
})
html = render(lv)
assert html =~ "must start with /"
end
test "clicking Remove deletes the rule", %{conn: conn, site: site} do
{:ok, _} =
Shields.add_page_rule(site, %{"page_path" => "/test/*/page"})
lv = get_liveview(conn, site)
html = render(lv)
assert html =~ "/test/*/page"
lv |> element(~s/button[phx-click="remove-page-rule"]/) |> render_click()
html = render(lv)
refute html =~ "/test/*/page"
assert Shields.count_page_rules(site) == 0
end
test "conclicting rules are annotated with a warning", %{conn: conn, site: site} do
lv = get_liveview(conn, site)
lv
|> element("form")
|> render_submit(%{
"page_rule[page_path]" => "/test/*"
})
html = render(lv)
refute html =~ "This rule might be redundant"
lv
|> element("form")
|> render_submit(%{
"page_rule[page_path]" => "/test/another"
})
html = render(lv)
assert html =~ "/test/*"
assert html =~ "/test/another"
assert html =~
"This rule might be redundant because the following rules may match first:\n\n/test/*"
broader_rule_id =
site
|> Shields.list_page_rules()
|> Enum.find(&(&1.page_path == "/test/*"))
|> Map.fetch!(:id)
lv |> element(~s/button#remove-page-rule-#{broader_rule_id}/) |> render_click()
html = render(lv)
assert html =~ "/test/another"
refute html =~ "/test/*"
refute html =~
"This rule might be redundant because the following rules may match first:\n\n/test/*"
end
defp get_liveview(conn, site) do
conn = assign(conn, :live_module, PlausibleWeb.Live.Shields)
{:ok, lv, _html} = live(conn, "/#{site.domain}/settings/shields/pages")
lv
end
end
end

@ -3,6 +3,7 @@ Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:window_time_on_page)
FunWithFlags.enable(:imports_exports)
FunWithFlags.enable(:shield_pages)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
if Mix.env() == :small_test do

Loading…
Cancel
Save