From 53f94a9f824a60292a300b8eac9a43ea22818dd2 Mon Sep 17 00:00:00 2001 From: hq1 Date: Mon, 25 Mar 2024 09:48:56 +0100 Subject: [PATCH 1/9] Pages shield (#3918) * Migration: Shield page rules * Add Ecto schema for Page Rules * Add Page Rule cache * Fix typo * BTW: Use already imported function * Extend Shields context interface + split existing tests * Ingestion: filter matching patches + refactor shield actions * Add LV section for adding Page Rules * Validate max page path length * Put Pages Shield behind a feature flag * Update CHANGELOG * Update docs link anchor As per https://github.com/plausible/docs/pull/477 * Update lib/plausible/shields.ex Co-authored-by: Adrian Gruntkowski * Update lib/plausible_web/live/shields/page_rules.ex Co-authored-by: ruslandoga * Update lib/plausible_web/live/shields/page_rules.ex Co-authored-by: ruslandoga --------- Co-authored-by: Adrian Gruntkowski Co-authored-by: ruslandoga --- CHANGELOG.md | 2 + lib/plausible/application.ex | 15 + lib/plausible/ingestion/event.ex | 31 +- lib/plausible/shield/country_rule.ex | 2 +- lib/plausible/shield/page_rule.ex | 85 +++++ lib/plausible/shield/page_rule_cache.ex | 65 ++++ lib/plausible/shields.ex | 74 +++++ lib/plausible/shields/country_test.exs | 136 ++++++++ lib/plausible/shields/ip_test.exs | 147 +++++++++ lib/plausible/shields/page_test.exs | 174 ++++++++++ .../controllers/site_controller.ex | 2 +- lib/plausible_web/live/shields/page_rules.ex | 304 ++++++++++++++++++ lib/plausible_web/live/shields/pages.ex | 53 +++ .../templates/site/settings_shields.html.heex | 7 + lib/plausible_web/views/layout_view.ex | 13 +- .../20240307083402_shield_page_rules.exs | 17 + priv/repo/seeds.exs | 1 + test/plausible/ingestion/event_test.exs | 19 ++ .../plausible/shield/page_rule_cache_test.exs | 107 ++++++ test/plausible/shields_test.exs | 270 ---------------- .../live/shields/ip_addresses_test.exs | 2 +- .../plausible_web/live/shields/pages_test.exs | 176 ++++++++++ test/test_helper.exs | 1 + 23 files changed, 1412 insertions(+), 291 deletions(-) create mode 100644 lib/plausible/shield/page_rule.ex create mode 100644 lib/plausible/shield/page_rule_cache.ex create mode 100644 lib/plausible/shields/country_test.exs create mode 100644 lib/plausible/shields/ip_test.exs create mode 100644 lib/plausible/shields/page_test.exs create mode 100644 lib/plausible_web/live/shields/page_rules.ex create mode 100644 lib/plausible_web/live/shields/pages.ex create mode 100644 priv/repo/migrations/20240307083402_shield_page_rules.exs create mode 100644 test/plausible/shield/page_rule_cache_test.exs create mode 100644 test/plausible_web/live/shields/pages_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed59f16c..a9263eb8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ### Added +- Pages Block List in Site Settings +- Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export - Ability to display total conversions (with a goal filter) on the main graph - Add `conversion_rate` to Stats API Timeseries and on the main graph diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 3f8117f42..f197ec6a0 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -77,6 +77,21 @@ defmodule Plausible.Application do interval: :timer.seconds(35), warmer_fn: :refresh_updated_recently ]}, + {Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.PageRuleCache.All, + cache_impl: Plausible.Shield.PageRuleCache, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), + warmer_fn: :refresh_all + ]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated, + cache_impl: Plausible.Shield.PageRuleCache, + interval: :timer.seconds(35), + warmer_fn: :refresh_updated_recently + ]}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, PlausibleWeb.Endpoint, {Oban, Application.get_env(:plausible, Oban)}, diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 8c66c01d7..6dd680f61 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -29,6 +29,7 @@ defmodule Plausible.Ingestion.Event do | :dc_ip | :site_ip_blocklist | :site_country_blocklist + | :site_page_blocklist @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -120,6 +121,7 @@ defmodule Plausible.Ingestion.Event do [ &drop_datacenter_ip/1, &drop_shield_rule_ip/1, + &drop_shield_rule_page/1, &put_geolocation/1, &drop_shield_rule_country/1, &put_user_agent/1, @@ -182,15 +184,18 @@ defmodule Plausible.Ingestion.Event do end defp drop_shield_rule_ip(%__MODULE__{} = event) do - domain = event.domain - address = event.request.remote_ip - - case Plausible.Shield.IPRuleCache.get({domain, address}) do - %Plausible.Shield.IPRule{action: :deny} -> - drop(event, :site_ip_blocklist) + if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do + drop(event, :site_ip_blocklist) + else + event + end + end - _ -> - event + defp drop_shield_rule_page(%__MODULE__{} = event) do + if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do + drop(event, :site_page_blocklist) + else + event end end @@ -263,12 +268,10 @@ defmodule Plausible.Ingestion.Event do %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event ) when is_binary(domain) and is_binary(cc) do - case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do - %Plausible.Shield.CountryRule{action: :deny} -> - drop(event, :site_country_blocklist) - - _ -> - event + if Plausible.Shields.country_blocked?(domain, cc) do + drop(event, :site_country_blocklist) + else + event end end diff --git a/lib/plausible/shield/country_rule.ex b/lib/plausible/shield/country_rule.ex index 85bc73456..067928f34 100644 --- a/lib/plausible/shield/country_rule.ex +++ b/lib/plausible/shield/country_rule.ex @@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do |> cast(attrs, [:site_id, :country_code]) |> validate_required([:site_id, :country_code]) |> validate_length(:country_code, is: 2) - |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc -> + |> validate_change(:country_code, fn :country_code, cc -> if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do [] else diff --git a/lib/plausible/shield/page_rule.ex b/lib/plausible/shield/page_rule.ex new file mode 100644 index 000000000..e874e185c --- /dev/null +++ b/lib/plausible/shield/page_rule.ex @@ -0,0 +1,85 @@ +defmodule CompiledRegex do + @moduledoc """ + Ensures that the regex is compiled on load + """ + use Ecto.Type + + def type, do: :string + + def cast(val) when is_binary(val), do: {:ok, val} + def cast(_), do: :error + + def load(val), do: {:ok, Regex.compile!(val)} + def dump(val), do: {:ok, val} +end + +defmodule Plausible.Shield.PageRule do + @moduledoc """ + Schema for Pages block list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_page" do + belongs_to :site, Plausible.Site + field :page_path, :string + field :page_path_pattern, CompiledRegex + field :action, Ecto.Enum, values: [:deny, :allow], default: :deny + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule \\ %__MODULE__{}, attrs) do + rule + |> cast(attrs, [:site_id, :page_path]) + |> validate_required([:site_id, :page_path]) + |> validate_length(:page_path, max: 250) + |> validate_change(:page_path, fn :page_path, p -> + if not String.starts_with?(p, "/") do + [page_path: "must start with /"] + else + [] + end + end) + |> store_regex() + |> unique_constraint(:page_path_pattern, + name: :shield_rules_page_site_id_page_path_pattern_index, + error_key: :page_path, + message: "rule already exists" + ) + end + + defp store_regex(changeset) do + case get_field(changeset, :page_path) do + "/" <> _ = page_path -> + regex = + page_path + |> Regex.escape() + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + regex = "^#{regex}$" + + verify_valid_regex(changeset, regex) + + _ -> + changeset + end + end + + defp verify_valid_regex(changeset, regex) do + case Regex.compile(regex) do + {:ok, _} -> + put_change(changeset, :page_path_pattern, regex) + + {:error, _} -> + add_error(changeset, :page_path, "could not compile regular expression") + end + end +end diff --git a/lib/plausible/shield/page_rule_cache.ex b/lib/plausible/shield/page_rule_cache.ex new file mode 100644 index 000000000..e9ba3039d --- /dev/null +++ b/lib/plausible/shield/page_rule_cache.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Shield.PageRuleCache do + @moduledoc """ + Allows retrieving Page Rules by domain + """ + alias Plausible.Shield.PageRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :page_blocklist_by_domain + + @cached_schema_fields ~w( + id + page_path_pattern + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_page_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(PageRule, :count) + end + + @impl true + def base_db_query() do + from rule in PageRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source(domain) do + query = + base_db_query() + |> where([..., site], site.domain == ^domain) + + case Plausible.Repo.one(query) do + {_, _, rule} -> %PageRule{rule | from_cache?: false} + _any -> nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {domain, object}, + {domain_changed_from, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shields.ex b/lib/plausible/shields.ex index 25ba47d84..942b1fba0 100644 --- a/lib/plausible/shields.ex +++ b/lib/plausible/shields.ex @@ -13,11 +13,63 @@ defmodule Plausible.Shields do @maximum_country_rules 30 def maximum_country_rules(), do: @maximum_country_rules + @maximum_page_rules 30 + def maximum_page_rules(), do: @maximum_page_rules + @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] def list_ip_rules(site_or_id) do list(Shield.IPRule, site_or_id) end + @spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def ip_blocked?(%Site{domain: domain}, address) do + ip_blocked?(domain, address) + end + + def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do + case Shield.IPRuleCache.get({domain, address}) do + %Shield.IPRule{action: :deny} -> + true + + _ -> + false + end + end + + @spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def page_blocked?(%Site{domain: domain}, address) do + page_blocked?(domain, address) + end + + def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do + page_rules = Shield.PageRuleCache.get(domain) + + if page_rules do + page_rules + |> List.wrap() + |> Enum.find_value(false, fn rule -> + rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname) + end) + else + false + end + end + + @spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def country_blocked?(%Site{domain: domain}, country_code) do + country_blocked?(domain, country_code) + end + + def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do + case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do + %Shield.CountryRule{action: :deny} -> + true + + _ -> + false + end + end + @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} def add_ip_rule(site_or_id, params, opts \\ []) do @@ -59,6 +111,28 @@ defmodule Plausible.Shields do count(Shield.CountryRule, site_or_id) end + @spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()] + def list_page_rules(site_or_id) do + list(Shield.PageRule, site_or_id) + end + + @spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()} + def add_page_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules}) + add(Shield.PageRule, site_or_id, params, opts) + end + + @spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_page_rule(site_or_id, rule_id) do + remove(Shield.PageRule, site_or_id, rule_id) + end + + @spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_page_rules(site_or_id) do + count(Shield.PageRule, site_or_id) + end + defp list(schema, %Site{id: id}) do list(schema, id) end diff --git a/lib/plausible/shields/country_test.exs b/lib/plausible/shields/country_test.exs new file mode 100644 index 000000000..8b2218688 --- /dev/null +++ b/lib/plausible/shields/country_test.exs @@ -0,0 +1,136 @@ +defmodule Plausible.Shields.CountryTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_country_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{}) + assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported country", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) + assert changeset.errors == [country_code: {"is invalid", []}] + refute changeset.valid? + end + + test "incorrect country format", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) + + assert changeset.errors == + [ + {:country_code, {"is invalid", []}}, + {:country_code, + {"should be %{count} character(s)", + [count: 2, validation: :length, kind: :is, type: :string]}} + ] + + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) + refute changeset.valid? + + assert changeset.errors == [ + country_code: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_country_site_id_country_code_index"} + ]} + ] + end + + test "over limit", %{site: site} do + country_codes = + Location.Country.all() + |> Enum.take(Plausible.Shields.maximum_country_rules()) + |> Enum.map(& &1.alpha_2) + + for cc <- country_codes do + assert {:ok, _} = + add_country_rule(site, %{"country_code" => cc}) + end + + assert count_country_rules(site) == maximum_country_rules() + + assert {:error, changeset} = + add_country_rule(site, %{"country_code" => "US"}) + + refute changeset.valid? + assert changeset.errors == [country_code: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_country_rule(site, %{"country_code" => "EE"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_country_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) + assert remove_country_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.CountryRule, rule.id) + assert remove_country_rule(site, rule.id) == :ok + end + end + + describe "list_country_rules/1" do + test "empty", %{site: site} do + assert(list_country_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) + :timer.sleep(1000) + {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) + assert [^r2, ^r1] = list_country_rules(site) + end + end + + describe "count_country_rules/1" do + test "counts", %{site: site} do + assert count_country_rules(site) == 0 + {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert count_country_rules(site) == 1 + {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) + assert count_country_rules(site) == 2 + end + end + + describe "Country Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_country_rules(site.id) == 0 + assert list_country_rules(site.id) == [] + + assert {:ok, rule} = + add_country_rule(site.id, %{"country_code" => "EE"}) + + add_country_rule(site2, %{"country_code" => "PL"}) + + assert count_country_rules(site) == 1 + assert [^rule] = list_country_rules(site) + assert rule.country_code == "EE" + assert rule.action == :deny + refute rule.from_cache? + assert country_blocked?(site, "ee") + refute country_blocked?(site, "xx") + end + end +end diff --git a/lib/plausible/shields/ip_test.exs b/lib/plausible/shields/ip_test.exs new file mode 100644 index 000000000..bd597a392 --- /dev/null +++ b/lib/plausible/shields/ip_test.exs @@ -0,0 +1,147 @@ +defmodule Plausible.Shields.IPTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_ip_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{}) + assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported netmask", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) + assert changeset.errors == [inet: {"netmask unsupported", []}] + refute changeset.valid? + assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) + end + + test "incorrect ip", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) + + assert changeset.errors == [ + inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} + ] + + refute changeset.valid? + end + + test "non-strict IPs", %{site: site} do + assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + refute changeset.valid? + + assert changeset.errors == [ + inet: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_ip_site_id_inet_index"} + ]} + ] + end + + test "ipv6", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "ipv4", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "over limit", %{site: site} do + for i <- 1..maximum_ip_rules() do + assert {:ok, _} = + add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) + end + + assert count_ip_rules(site) == maximum_ip_rules() + + assert {:error, changeset} = + add_ip_rule(site, %{"inet" => "1.1.1.31"}) + + refute changeset.valid? + assert changeset.errors == [inet: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_ip_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert remove_ip_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.IPRule, rule.id) + assert remove_ip_rule(site, rule.id) == :ok + end + end + + describe "list_ip_rules/1" do + test "empty", %{site: site} do + assert(list_ip_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + :timer.sleep(1000) + {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert [^r2, ^r1] = list_ip_rules(site) + end + end + + describe "count_ip_rules/1" do + test "counts", %{site: site} do + assert count_ip_rules(site) == 0 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert count_ip_rules(site) == 1 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert count_ip_rules(site) == 2 + end + end + + describe "IP Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_ip_rules(site.id) == 0 + assert list_ip_rules(site.id) == [] + + assert {:ok, rule} = + add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + assert count_ip_rules(site) == 1 + assert [^rule] = list_ip_rules(site) + assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} + assert rule.description == "Localhost" + assert rule.action == :deny + refute rule.from_cache? + assert ip_blocked?(site, "127.0.0.1") + refute ip_blocked?(site, "127.0.0.2") + end + end +end diff --git a/lib/plausible/shields/page_test.exs b/lib/plausible/shields/page_test.exs new file mode 100644 index 000000000..6134e9903 --- /dev/null +++ b/lib/plausible/shields/page_test.exs @@ -0,0 +1,174 @@ +defmodule Plausible.Shields.PageTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_page_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_page_rule(site, %{}) + assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "no slash", %{site: site} do + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"}) + assert changeset.errors == [page_path: {"must start with /", []}] + refute changeset.valid? + end + + test "lengthy", %{site: site} do + long = "/" <> :binary.copy("a", 251) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => long}) + assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"}) + refute changeset.valid? + + assert changeset.errors == [ + page_path: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} + ]} + ] + end + + test "equivalent rules are counted as dupes", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"}) + + assert changeset.errors == [ + page_path: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} + ]} + ] + end + + test "regex storage: wildcard", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"}) + assert rule.page_path_pattern == "^/test/.*$" + end + + test "regex storage: no wildcard", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) + assert rule.page_path_pattern == "^/test$" + end + + test "regex storage: escaping", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"}) + assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$" + end + + test "over limit", %{site: site} do + for i <- 1..maximum_page_rules() do + assert {:ok, _} = + add_page_rule(site, %{"page_path" => "/test/#{i}"}) + end + + assert count_page_rules(site) == maximum_page_rules() + + assert {:error, changeset} = + add_page_rule(site, %{"page_path" => "/test/31"}) + + refute changeset.valid? + assert changeset.errors == [page_path: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_page_rule(site, %{"page_path" => "/test"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "page pattern matching" do + test "no wildcard", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) + assert page_blocked?(site, "/test") + refute page_blocked?(site, "/test/hello") + refute page_blocked?(site, "test") + end + + test "wildcard", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) + refute page_blocked?(site, "/test") + assert page_blocked?(site, "/test/") + assert page_blocked?(site, "/test/hello") + refute page_blocked?(site, "test") + refute page_blocked?(site, "/testing") + end + end + + describe "remove_page_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) + assert remove_page_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.PageRule, rule.id) + assert remove_page_rule(site, rule.id) == :ok + end + end + + describe "list_page_rules/1" do + test "empty", %{site: site} do + assert(list_page_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_page_rule(site, %{"page_path" => "/test"}) + :timer.sleep(1000) + {:ok, r2} = add_page_rule(site, %{"page_path" => "/test"}) + assert [^r2, ^r1] = list_page_rules(site) + end + end + + describe "count_page_rules/1" do + test "counts", %{site: site} do + assert count_page_rules(site) == 0 + {:ok, _} = add_page_rule(site, %{"page_path" => "/test1"}) + assert count_page_rules(site) == 1 + {:ok, _} = add_page_rule(site, %{"page_path" => "/test2"}) + assert count_page_rules(site) == 2 + end + end + + describe "Page Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_page_rules(site.id) == 0 + assert list_page_rules(site.id) == [] + + assert {:ok, rule} = + add_page_rule(site.id, %{ + "page_path" => "/test" + }) + + add_page_rule(site2, %{"page_path" => "/test"}) + + assert count_page_rules(site) == 1 + assert [%{id: rule_id}] = list_page_rules(site) + assert rule.id == rule_id + assert rule.page_path == "/test" + assert rule.action == :deny + refute rule.from_cache? + assert page_blocked?(site, "/test") + refute page_blocked?(site, "/testing") + end + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 436c391c7..77c9a9343 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -257,7 +257,7 @@ defmodule PlausibleWeb.SiteController do end def settings_shields(conn, %{"shield" => shield}) - when shield in ["ip_addresses", "countries"] do + when shield in ["ip_addresses", "countries", "pages"] do site = conn.assigns.site conn diff --git a/lib/plausible_web/live/shields/page_rules.ex b/lib/plausible_web/live/shields/page_rules.ex new file mode 100644 index 000000000..3ff38e12b --- /dev/null +++ b/lib/plausible_web/live/shields/page_rules.ex @@ -0,0 +1,304 @@ +defmodule PlausibleWeb.Live.Shields.PageRules do + @moduledoc """ + LiveView allowing page Rules management + """ + + use Phoenix.LiveComponent, global_prefixes: ~w(x-) + use Phoenix.HTML + + alias PlausibleWeb.Live.Components.Modal + alias Plausible.Shields + alias Plausible.Shield + + import PlausibleWeb.ErrorHelpers + + def update(assigns, socket) do + socket = + socket + |> assign( + page_rules_count: assigns[:page_rules_count] || socket.assigns.page_rules_count, + site: assigns[:site] || socket.assigns.site, + current_user: assigns[:current_user] || socket.assigns.current_user, + form: new_form() + ) + |> assign_new(:page_rules, fn %{site: site} -> + Shields.list_page_rules(site) + end) + |> assign_new(:redundant_rules, fn %{page_rules: page_rules} -> + detect_redundancy(page_rules) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+
+
+

+ Pages Block List +

+

+ Reject incoming traffic for specific pages +

+ + +
+
+
+ + + Add Page + +
+ = Shields.maximum_page_rules()} + class="mt-4" + title="Maximum number of pages reached" + > +

+ You've reached the maximum number of pages you can block (<%= Shields.maximum_page_rules() %>). Please remove one before adding another. +

+
+
+ + <.live_component module={Modal} id="page-rule-form-modal"> + <.form + :let={f} + for={@form} + phx-submit="save-page-rule" + phx-target={@myself} + class="max-w-md w-full mx-auto bg-white dark:bg-gray-800" + > +

Add Page to Block List

+ + <.live_component + submit_name="page_rule[page_path]" + submit_value={f[:page_path].value} + display_value={f[:page_path].value || ""} + module={PlausibleWeb.Live.Components.ComboBox} + suggest_fun={fn input, options -> suggest_page_paths(input, options, @site) end} + id={f[:page_path].id} + creatable + /> + + <%= error_tag(f, :page_path) %> + +

+ You can use a wildcard (*) to match multiple pages. For example, + /blog/* + will match /blog/post. + Once added, we will start rejecting traffic from this page within a few minutes. +

+
+ + Add Page → + +
+ + + +

+ No Page Rules configured for this Site. +

+
+ + + + + + + + + + <%= for rule <- @page_rules do %> + + + + + + + <% end %> + +
+ page + + Status + + Remove +
+
+ + <%= rule.page_path %> + +
+
+
+ + Blocked + + + Allowed + + + + + +
+
+ +
+
+
+
+ """ + end + + def handle_event("save-page-rule", %{"page_rule" => params}, socket) do + user = socket.assigns.current_user + + case Shields.add_page_rule( + socket.assigns.site.id, + params, + added_by: user + ) do + {:ok, rule} -> + page_rules = [rule | socket.assigns.page_rules] + + socket = + socket + |> Modal.close("page-rule-form-modal") + |> assign( + form: new_form(), + page_rules: page_rules, + page_rules_count: socket.assigns.page_rules_count + 1, + redundant_rules: detect_redundancy(page_rules) + ) + + # Make sure to clear the combobox input after adding a page rule, on subsequent modal reopening + send_update(PlausibleWeb.Live.Components.ComboBox, + id: "page_rule_page_code", + display_value: "" + ) + + send_flash( + :success, + "Page rule added successfully. Traffic will be rejected within a few minutes." + ) + + {:noreply, socket} + + {:error, %Ecto.Changeset{} = changeset} -> + {:noreply, assign(socket, form: to_form(changeset))} + end + end + + def handle_event("remove-page-rule", %{"rule-id" => rule_id}, socket) do + Shields.remove_page_rule(socket.assigns.site.id, rule_id) + + send_flash( + :success, + "Page rule removed successfully. Traffic will be resumed within a few minutes." + ) + + page_rules = Enum.reject(socket.assigns.page_rules, &(&1.id == rule_id)) + + {:noreply, + socket + |> assign( + page_rules_count: socket.assigns.page_rules_count - 1, + page_rules: page_rules, + redundant_rules: detect_redundancy(page_rules) + )} + end + + def send_flash(kind, message) do + send(self(), {:flash, kind, message}) + end + + defp new_form() do + %Shield.PageRule{} + |> Shield.PageRule.changeset(%{}) + |> to_form() + end + + defp format_added_at(dt, tz) do + dt + |> Plausible.Timezones.to_datetime_in_timezone(tz) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") + end + + def suggest_page_paths(input, _options, site) do + query = Plausible.Stats.Query.from(site, %{}) + + site + |> Plausible.Stats.filter_suggestions(query, "page", input) + |> Enum.map(fn %{label: label, value: value} -> {label, value} end) + end + + defp detect_redundancy(page_rules) do + page_rules + |> Enum.reduce(%{}, fn rule, acc -> + {[^rule], remaining_rules} = + Enum.split_with( + page_rules, + fn r -> r == rule end + ) + + conflicting = + remaining_rules + |> Enum.filter(fn candidate -> + rule + |> Map.fetch!(:page_path_pattern) + |> maybe_compile() + |> Regex.match?(candidate.page_path) + end) + |> Enum.map(& &1.id) + + Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc -> + Map.update(acc, conflicting_rule_id, [rule.page_path], fn existing -> + [rule.page_path | existing] + end) + end) + end) + end + + defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern) + defp maybe_compile(%Regex{} = pattern), do: pattern +end diff --git a/lib/plausible_web/live/shields/pages.ex b/lib/plausible_web/live/shields/pages.ex new file mode 100644 index 000000000..cd42c47b3 --- /dev/null +++ b/lib/plausible_web/live/shields/pages.ex @@ -0,0 +1,53 @@ +defmodule PlausibleWeb.Live.Shields.Pages do + @moduledoc """ + LiveView for IP Addresses Shield + """ + use PlausibleWeb, :live_view + use Phoenix.HTML + + alias Plausible.Shields + alias Plausible.Sites + + def mount( + _params, + %{ + "domain" => domain, + "current_user_id" => user_id + }, + socket + ) do + socket = + socket + |> assign_new(:site, fn -> + Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin]) + end) + |> assign_new(:page_rules_count, fn %{site: site} -> + Shields.count_page_rules(site) + end) + |> assign_new(:current_user, fn -> + Plausible.Repo.get(Plausible.Auth.User, user_id) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+ <.flash_messages flash={@flash} /> + <.live_component + module={PlausibleWeb.Live.Shields.PageRules} + current_user={@current_user} + page_rules_count={@page_rules_count} + site={@site} + id="page-rules-#{@current_user.id}" + /> +
+ """ + end + + def handle_info({:flash, kind, message}, socket) do + socket = put_live_flash(socket, kind, message) + {:noreply, socket} + end +end diff --git a/lib/plausible_web/templates/site/settings_shields.html.heex b/lib/plausible_web/templates/site/settings_shields.html.heex index 411c39fd3..489d49daf 100644 --- a/lib/plausible_web/templates/site/settings_shields.html.heex +++ b/lib/plausible_web/templates/site/settings_shields.html.heex @@ -14,4 +14,11 @@ "domain" => @site.domain } ) %> + <% "pages" -> %> + <%= live_render(@conn, PlausibleWeb.Live.Shields.Pages, + session: %{ + "site_id" => @site.id, + "domain" => @site.domain + } + ) %> <% end %> diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index 0b587379d..42923e6f2 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -65,10 +65,15 @@ defmodule PlausibleWeb.LayoutView do %{ key: "Shields", icon: :shield_exclamation, - value: [ - %{key: "IP Addresses", value: "shields/ip_addresses"}, - %{key: "Countries", value: "shields/countries"} - ] + value: + [ + %{key: "IP Addresses", value: "shields/ip_addresses"}, + %{key: "Countries", value: "shields/countries"}, + if FunWithFlags.enabled?(:shield_pages) do + %{key: "Pages", value: "shields/pages"} + end + ] + |> Enum.reject(&is_nil/1) }, %{key: "Email Reports", value: "email-reports", icon: :envelope}, if conn.assigns[:current_user_role] == :owner do diff --git a/priv/repo/migrations/20240307083402_shield_page_rules.exs b/priv/repo/migrations/20240307083402_shield_page_rules.exs new file mode 100644 index 000000000..cd5ee1a41 --- /dev/null +++ b/priv/repo/migrations/20240307083402_shield_page_rules.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.ShieldPageRules do + use Ecto.Migration + + def change do + create table(:shield_rules_page, primary_key: false) do + add(:id, :uuid, primary_key: true) + add :site_id, references(:sites, on_delete: :delete_all), null: false + add :page_path, :text, null: false + add :page_path_pattern, :text, null: false + add :action, :string, default: "deny", null: false + add :added_by, :string + timestamps() + end + + create unique_index(:shield_rules_page, [:site_id, :page_path_pattern]) + end +end diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index e1b499f76..da7b842d9 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -11,6 +11,7 @@ # and so on) as they will fail if something goes wrong. FunWithFlags.enable(:imports_exports) +FunWithFlags.enable(:shield_pages) user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index b6627d8bc..8714e0848 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -158,6 +158,25 @@ defmodule Plausible.Ingestion.EventTest do assert dropped.drop_reason == :site_country_blocklist end + test "event pipeline drops a request when page is on blocklist" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site/blocked/page", + domain: site.domain + } + + conn = build_conn(:post, "/api/events", payload) + + {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/blocked/**"}) + + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :site_page_blocklist + end + test "event pipeline drops events for site with accept_trafic_until in the past" do yesterday = Date.add(Date.utc_today(), -1) diff --git a/test/plausible/shield/page_rule_cache_test.exs b/test/plausible/shield/page_rule_cache_test.exs new file mode 100644 index 000000000..ba26f3cf9 --- /dev/null +++ b/test/plausible/shield/page_rule_cache_test.exs @@ -0,0 +1,107 @@ +defmodule Plausible.Shield.PageRuleCacheTest do + use Plausible.DataCase, async: true + + alias Plausible.Shield.PageRule + alias Plausible.Shield.PageRuleCache + alias Plausible.Shields + + describe "public cache interface" do + test "cache caches page rules", %{test: test} do + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = + Supervisor.start_link( + [ + {PageRuleCache, + [cache_name: test, child_id: :page_rules_cache_id, ets_options: [:bag]]} + ], + strategy: :one_for_one, + name: :"cache_supervisor_#{test}" + ) + + site1 = insert(:site) + site2 = insert(:site) + + {:ok, %{id: rid1}} = Shields.add_page_rule(site1, %{"page_path" => "/test/1"}) + {:ok, %{id: rid2}} = Shields.add_page_rule(site1, %{"page_path" => "/test/2"}) + {:ok, %{id: rid3}} = Shields.add_page_rule(site2, %{"page_path" => "/test/2"}) + + :ok = PageRuleCache.refresh_all(cache_name: test) + + :ok = Shields.remove_page_rule(site1, rid1) + + assert PageRuleCache.size(test) == 3 + + # the rule order should be deterministic, but with 1s timestamp sorting precision + # race conditions may happen during tests + assert rules = PageRuleCache.get(site1.domain, cache_opts) + rule_ids = Enum.map(rules, & &1.id) + assert rid1 in rule_ids + assert rid2 in rule_ids + assert length(rules) == 2 + + assert %PageRule{from_cache?: true, id: ^rid3} = + PageRuleCache.get(site2.domain, cache_opts) + + refute PageRuleCache.get("rogue.example.com", cache_opts) + end + + test "page path patterns are already compiled when fetched from cache", %{test: test} do + site = insert(:site) + + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/hello/**/world"}) + :ok = PageRuleCache.refresh_all(cache_name: test) + assert regex = PageRuleCache.get(site.domain, cache_opts).page_path_pattern + assert regex == ~r/^\/hello\/.*\/world$/ + end + + test "cache allows lookups for page paths on sites with changed domain", %{test: test} do + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + site = insert(:site, domain: "new.example.com", domain_changed_from: "old.example.com") + + {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/#{test}"}) + :ok = PageRuleCache.refresh_all(cache_name: test) + + assert PageRuleCache.get("old.example.com", cache_opts) + assert PageRuleCache.get("new.example.com", cache_opts) + refute PageRuleCache.get("rogue.example.com", cache_opts) + end + + test "refreshes only recently added pages rules", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain = "site1.example.com" + site = insert(:site, domain: domain) + + cache_opts = [cache_name: test, force?: true] + + yesterday = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-1 * 60 * 60 * 24) + |> NaiveDateTime.truncate(:second) + + {:ok, r1} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/1"}) + Ecto.Changeset.change(r1, inserted_at: yesterday, updated_at: yesterday) |> Repo.update!() + {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/2"}) + + assert PageRuleCache.get(domain, cache_opts) == nil + + assert :ok = PageRuleCache.refresh_updated_recently(cache_opts) + + assert %{page_path_pattern: ~r[^/test/2$]} = PageRuleCache.get(domain, cache_opts) + + assert :ok = PageRuleCache.refresh_all(cache_opts) + + assert [_, _] = PageRuleCache.get(domain, cache_opts) + end + end + + defp start_test_cache(cache_name) do + %{start: {m, f, a}} = PageRuleCache.child_spec(cache_name: cache_name, ets_options: [:bag]) + apply(m, f, a) + end +end diff --git a/test/plausible/shields_test.exs b/test/plausible/shields_test.exs index c39da4587..e69de29bb 100644 --- a/test/plausible/shields_test.exs +++ b/test/plausible/shields_test.exs @@ -1,270 +0,0 @@ -defmodule Plausible.ShieldsTest do - use Plausible.DataCase - import Plausible.Shields - - setup do - site = insert(:site) - {:ok, %{site: site}} - end - - describe "add_ip_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{}) - assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported netmask", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) - assert changeset.errors == [inet: {"netmask unsupported", []}] - refute changeset.valid? - assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) - end - - test "incorrect ip", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) - - assert changeset.errors == [ - inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} - ] - - refute changeset.valid? - end - - test "non-strict IPs", %{site: site} do - assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - refute changeset.valid? - - assert changeset.errors == [ - inet: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_ip_site_id_inet_index"} - ]} - ] - end - - test "ipv6", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "ipv4", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "over limit", %{site: site} do - for i <- 1..maximum_ip_rules() do - assert {:ok, _} = - add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) - end - - assert count_ip_rules(site) == maximum_ip_rules() - - assert {:error, changeset} = - add_ip_rule(site, %{"inet" => "1.1.1.31"}) - - refute changeset.valid? - assert changeset.errors == [inet: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_ip_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert remove_ip_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.IPRule, rule.id) - assert remove_ip_rule(site, rule.id) == :ok - end - end - - describe "list_ip_rules/1" do - test "empty", %{site: site} do - assert(list_ip_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - :timer.sleep(1000) - {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert [^r2, ^r1] = list_ip_rules(site) - end - end - - describe "count_ip_rules/1" do - test "counts", %{site: site} do - assert count_ip_rules(site) == 0 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert count_ip_rules(site) == 1 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert count_ip_rules(site) == 2 - end - end - - describe "IP Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_ip_rules(site.id) == 0 - assert list_ip_rules(site.id) == [] - - assert {:ok, rule} = - add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - assert count_ip_rules(site) == 1 - assert [^rule] = list_ip_rules(site) - assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} - assert rule.description == "Localhost" - assert rule.action == :deny - refute rule.from_cache? - end - end - - describe "add_country_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{}) - assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported country", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) - assert changeset.errors == [country_code: {"is invalid", []}] - refute changeset.valid? - end - - test "incorrect country format", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) - - assert changeset.errors == - [ - {:country_code, {"is invalid", []}}, - {:country_code, - {"should be %{count} character(s)", - [count: 2, validation: :length, kind: :is, type: :string]}} - ] - - refute changeset.valid? - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) - refute changeset.valid? - - assert changeset.errors == [ - country_code: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_country_site_id_country_code_index"} - ]} - ] - end - - test "over limit", %{site: site} do - country_codes = - Location.Country.all() - |> Enum.take(Plausible.Shields.maximum_country_rules()) - |> Enum.map(& &1.alpha_2) - - for cc <- country_codes do - assert {:ok, _} = - add_country_rule(site, %{"country_code" => cc}) - end - - assert count_country_rules(site) == maximum_country_rules() - - assert {:error, changeset} = - add_country_rule(site, %{"country_code" => "US"}) - - refute changeset.valid? - assert changeset.errors == [country_code: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_country_rule(site, %{"country_code" => "EE"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_country_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) - assert remove_country_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.CountryRule, rule.id) - assert remove_country_rule(site, rule.id) == :ok - end - end - - describe "list_country_rules/1" do - test "empty", %{site: site} do - assert(list_country_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) - :timer.sleep(1000) - {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) - assert [^r2, ^r1] = list_country_rules(site) - end - end - - describe "count_country_rules/1" do - test "counts", %{site: site} do - assert count_country_rules(site) == 0 - {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert count_country_rules(site) == 1 - {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) - assert count_country_rules(site) == 2 - end - end - - describe "Country Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_country_rules(site.id) == 0 - assert list_country_rules(site.id) == [] - - assert {:ok, rule} = - add_country_rule(site.id, %{"country_code" => "EE"}) - - add_country_rule(site2, %{"country_code" => "PL"}) - - assert count_country_rules(site) == 1 - assert [^rule] = list_country_rules(site) - assert rule.country_code == "EE" - assert rule.action == :deny - refute rule.from_cache? - end - end -end diff --git a/test/plausible_web/live/shields/ip_addresses_test.exs b/test/plausible_web/live/shields/ip_addresses_test.exs index fd11a431d..16a4f2b1a 100644 --- a/test/plausible_web/live/shields/ip_addresses_test.exs +++ b/test/plausible_web/live/shields/ip_addresses_test.exs @@ -95,7 +95,7 @@ defmodule PlausibleWeb.Live.Shields.IPAddressesTest do assert element_exists?(html, ~s/a[phx-click="prefill-own-ip-rule"]/) end - test "form modal does not contains link to add own IP if already added", %{ + test "form modal does not contain link to add own IP if already added", %{ site: site, conn: conn } do diff --git a/test/plausible_web/live/shields/pages_test.exs b/test/plausible_web/live/shields/pages_test.exs new file mode 100644 index 000000000..7f89456a8 --- /dev/null +++ b/test/plausible_web/live/shields/pages_test.exs @@ -0,0 +1,176 @@ +defmodule PlausibleWeb.Live.Shields.PagesTest do + use PlausibleWeb.ConnCase, async: true + + import Phoenix.LiveViewTest + import Plausible.Test.Support.HTML + + alias Plausible.Shields + + setup [:create_user, :create_site, :log_in] + + describe "Page Rules - static" do + test "renders page rules page with empty list", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert resp =~ "No Page Rules configured for this Site" + assert resp =~ "Pages Block List" + end + + test "lists page rules with remove actions", %{conn: conn, site: site} do + {:ok, r1} = + Shields.add_page_rule(site, %{"page_path" => "/test/1"}) + + {:ok, r2} = + Shields.add_page_rule(site, %{"page_path" => "/test/2"}) + + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert resp =~ "/test/1" + assert resp =~ "/test/2" + + assert remove_button_1 = find(resp, "#remove-page-rule-#{r1.id}") + assert remove_button_2 = find(resp, "#remove-page-rule-#{r2.id}") + + assert text_of_attr(remove_button_1, "phx-click" == "remove-page-rule") + assert text_of_attr(remove_button_1, "phx-value-rule-id" == r1.id) + assert text_of_attr(remove_button_2, "phx-click" == "remove-page-rule") + assert text_of_attr(remove_button_2, "phx-value-rule-id" == r2.id) + end + + test "add rule button is rendered", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert element_exists?(resp, ~s/button#add-page-rule[x-data]/) + attr = text_of_attr(resp, ~s/button#add-page-rule/, "x-on:click") + + assert attr =~ "open-modal" + assert attr =~ "page-rule-form-modal" + end + + test "add rule button is not rendered when maximum reached", %{conn: conn, site: site} do + for i <- 1..Shields.maximum_page_rules() do + assert {:ok, _} = + Shields.add_page_rule(site, %{"page_path" => "/test/#{i}"}) + end + + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + refute element_exists?(resp, ~s/button#add-page-rule[x-data]/) + assert resp =~ "Maximum number of pages reached" + assert resp =~ "You've reached the maximum number of pages you can block (30)" + end + end + + describe "Page Rules - LiveView" do + test "modal contains form", %{site: site, conn: conn} do + lv = get_liveview(conn, site) + html = render(lv) + + assert element_exists?( + html, + ~s/form[phx-submit="save-page-rule"] input[name="page_rule\[page_path\]"]/ + ) + + assert submit_button(html, ~s/form[phx-submit="save-page-rule"]/) + end + + test "submitting a valid Page saves it", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/**" + }) + + html = render(lv) + + assert html =~ "/test/**" + + assert [%{page_path: "/test/**"}] = Shields.list_page_rules(site) + end + + test "submitting invalid Page renders error", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "WRONG" + }) + + html = render(lv) + assert html =~ "must start with /" + end + + test "clicking Remove deletes the rule", %{conn: conn, site: site} do + {:ok, _} = + Shields.add_page_rule(site, %{"page_path" => "/test/*/page"}) + + lv = get_liveview(conn, site) + + html = render(lv) + assert html =~ "/test/*/page" + + lv |> element(~s/button[phx-click="remove-page-rule"]/) |> render_click() + + html = render(lv) + refute html =~ "/test/*/page" + + assert Shields.count_page_rules(site) == 0 + end + + test "conclicting rules are annotated with a warning", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/*" + }) + + html = render(lv) + refute html =~ "This rule might be redundant" + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/another" + }) + + html = render(lv) + + assert html =~ "/test/*" + assert html =~ "/test/another" + + assert html =~ + "This rule might be redundant because the following rules may match first:\n\n/test/*" + + broader_rule_id = + site + |> Shields.list_page_rules() + |> Enum.find(&(&1.page_path == "/test/*")) + |> Map.fetch!(:id) + + lv |> element(~s/button#remove-page-rule-#{broader_rule_id}/) |> render_click() + html = render(lv) + + assert html =~ "/test/another" + refute html =~ "/test/*" + + refute html =~ + "This rule might be redundant because the following rules may match first:\n\n/test/*" + end + + defp get_liveview(conn, site) do + conn = assign(conn, :live_module, PlausibleWeb.Live.Shields) + {:ok, lv, _html} = live(conn, "/#{site.domain}/settings/shields/pages") + + lv + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 75e431245..78171693a 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -3,6 +3,7 @@ Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) Application.ensure_all_started(:double) FunWithFlags.enable(:window_time_on_page) FunWithFlags.enable(:imports_exports) +FunWithFlags.enable(:shield_pages) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) if Mix.env() == :small_test do From 9989ce6927ef02a4d26e51e93f5301231bf42a71 Mon Sep 17 00:00:00 2001 From: hq1 Date: Mon, 25 Mar 2024 10:19:50 +0100 Subject: [PATCH 2/9] Migration for 3918 (#3939) * Revert "Pages shield (#3918)" This reverts commit 53f94a9f824a60292a300b8eac9a43ea22818dd2. * Migration: Shield page rules --- CHANGELOG.md | 2 - lib/plausible/application.ex | 15 - lib/plausible/ingestion/event.ex | 31 +- lib/plausible/shield/country_rule.ex | 2 +- lib/plausible/shield/page_rule.ex | 85 ----- lib/plausible/shield/page_rule_cache.ex | 65 ---- lib/plausible/shields.ex | 74 ----- lib/plausible/shields/country_test.exs | 136 -------- lib/plausible/shields/ip_test.exs | 147 --------- lib/plausible/shields/page_test.exs | 174 ---------- .../controllers/site_controller.ex | 2 +- lib/plausible_web/live/shields/page_rules.ex | 304 ------------------ lib/plausible_web/live/shields/pages.ex | 53 --- .../templates/site/settings_shields.html.heex | 7 - lib/plausible_web/views/layout_view.ex | 13 +- priv/repo/seeds.exs | 1 - test/plausible/ingestion/event_test.exs | 19 -- .../plausible/shield/page_rule_cache_test.exs | 107 ------ test/plausible/shields_test.exs | 270 ++++++++++++++++ .../live/shields/ip_addresses_test.exs | 2 +- .../plausible_web/live/shields/pages_test.exs | 176 ---------- test/test_helper.exs | 1 - 22 files changed, 291 insertions(+), 1395 deletions(-) delete mode 100644 lib/plausible/shield/page_rule.ex delete mode 100644 lib/plausible/shield/page_rule_cache.ex delete mode 100644 lib/plausible/shields/country_test.exs delete mode 100644 lib/plausible/shields/ip_test.exs delete mode 100644 lib/plausible/shields/page_test.exs delete mode 100644 lib/plausible_web/live/shields/page_rules.ex delete mode 100644 lib/plausible_web/live/shields/pages.ex delete mode 100644 test/plausible/shield/page_rule_cache_test.exs delete mode 100644 test/plausible_web/live/shields/pages_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index a9263eb8d..4ed59f16c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ All notable changes to this project will be documented in this file. ### Added -- Pages Block List in Site Settings -- Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export - Ability to display total conversions (with a goal filter) on the main graph - Add `conversion_rate` to Stats API Timeseries and on the main graph diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index f197ec6a0..3f8117f42 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -77,21 +77,6 @@ defmodule Plausible.Application do interval: :timer.seconds(35), warmer_fn: :refresh_updated_recently ]}, - {Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.PageRuleCache.All, - cache_impl: Plausible.Shield.PageRuleCache, - interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), - warmer_fn: :refresh_all - ]}, - {Plausible.Cache.Warmer, - [ - child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated, - cache_impl: Plausible.Shield.PageRuleCache, - interval: :timer.seconds(35), - warmer_fn: :refresh_updated_recently - ]}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, PlausibleWeb.Endpoint, {Oban, Application.get_env(:plausible, Oban)}, diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 6dd680f61..8c66c01d7 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -29,7 +29,6 @@ defmodule Plausible.Ingestion.Event do | :dc_ip | :site_ip_blocklist | :site_country_blocklist - | :site_page_blocklist @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -121,7 +120,6 @@ defmodule Plausible.Ingestion.Event do [ &drop_datacenter_ip/1, &drop_shield_rule_ip/1, - &drop_shield_rule_page/1, &put_geolocation/1, &drop_shield_rule_country/1, &put_user_agent/1, @@ -184,18 +182,15 @@ defmodule Plausible.Ingestion.Event do end defp drop_shield_rule_ip(%__MODULE__{} = event) do - if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do - drop(event, :site_ip_blocklist) - else - event - end - end + domain = event.domain + address = event.request.remote_ip - defp drop_shield_rule_page(%__MODULE__{} = event) do - if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do - drop(event, :site_page_blocklist) - else - event + case Plausible.Shield.IPRuleCache.get({domain, address}) do + %Plausible.Shield.IPRule{action: :deny} -> + drop(event, :site_ip_blocklist) + + _ -> + event end end @@ -268,10 +263,12 @@ defmodule Plausible.Ingestion.Event do %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event ) when is_binary(domain) and is_binary(cc) do - if Plausible.Shields.country_blocked?(domain, cc) do - drop(event, :site_country_blocklist) - else - event + case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do + %Plausible.Shield.CountryRule{action: :deny} -> + drop(event, :site_country_blocklist) + + _ -> + event end end diff --git a/lib/plausible/shield/country_rule.ex b/lib/plausible/shield/country_rule.ex index 067928f34..85bc73456 100644 --- a/lib/plausible/shield/country_rule.ex +++ b/lib/plausible/shield/country_rule.ex @@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do |> cast(attrs, [:site_id, :country_code]) |> validate_required([:site_id, :country_code]) |> validate_length(:country_code, is: 2) - |> validate_change(:country_code, fn :country_code, cc -> + |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc -> if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do [] else diff --git a/lib/plausible/shield/page_rule.ex b/lib/plausible/shield/page_rule.ex deleted file mode 100644 index e874e185c..000000000 --- a/lib/plausible/shield/page_rule.ex +++ /dev/null @@ -1,85 +0,0 @@ -defmodule CompiledRegex do - @moduledoc """ - Ensures that the regex is compiled on load - """ - use Ecto.Type - - def type, do: :string - - def cast(val) when is_binary(val), do: {:ok, val} - def cast(_), do: :error - - def load(val), do: {:ok, Regex.compile!(val)} - def dump(val), do: {:ok, val} -end - -defmodule Plausible.Shield.PageRule do - @moduledoc """ - Schema for Pages block list - """ - use Ecto.Schema - import Ecto.Changeset - - @type t() :: %__MODULE__{} - - @primary_key {:id, :binary_id, autogenerate: true} - schema "shield_rules_page" do - belongs_to :site, Plausible.Site - field :page_path, :string - field :page_path_pattern, CompiledRegex - field :action, Ecto.Enum, values: [:deny, :allow], default: :deny - field :added_by, :string - - # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` - field :from_cache?, :boolean, virtual: true, default: false - timestamps() - end - - def changeset(rule \\ %__MODULE__{}, attrs) do - rule - |> cast(attrs, [:site_id, :page_path]) - |> validate_required([:site_id, :page_path]) - |> validate_length(:page_path, max: 250) - |> validate_change(:page_path, fn :page_path, p -> - if not String.starts_with?(p, "/") do - [page_path: "must start with /"] - else - [] - end - end) - |> store_regex() - |> unique_constraint(:page_path_pattern, - name: :shield_rules_page_site_id_page_path_pattern_index, - error_key: :page_path, - message: "rule already exists" - ) - end - - defp store_regex(changeset) do - case get_field(changeset, :page_path) do - "/" <> _ = page_path -> - regex = - page_path - |> Regex.escape() - |> String.replace("\\*\\*", ".*") - |> String.replace("\\*", ".*") - - regex = "^#{regex}$" - - verify_valid_regex(changeset, regex) - - _ -> - changeset - end - end - - defp verify_valid_regex(changeset, regex) do - case Regex.compile(regex) do - {:ok, _} -> - put_change(changeset, :page_path_pattern, regex) - - {:error, _} -> - add_error(changeset, :page_path, "could not compile regular expression") - end - end -end diff --git a/lib/plausible/shield/page_rule_cache.ex b/lib/plausible/shield/page_rule_cache.ex deleted file mode 100644 index e9ba3039d..000000000 --- a/lib/plausible/shield/page_rule_cache.ex +++ /dev/null @@ -1,65 +0,0 @@ -defmodule Plausible.Shield.PageRuleCache do - @moduledoc """ - Allows retrieving Page Rules by domain - """ - alias Plausible.Shield.PageRule - - import Ecto.Query - use Plausible.Cache - - @cache_name :page_blocklist_by_domain - - @cached_schema_fields ~w( - id - page_path_pattern - action - )a - - @impl true - def name(), do: @cache_name - - @impl true - def child_id(), do: :cache_page_blocklist - - @impl true - def count_all() do - Plausible.Repo.aggregate(PageRule, :count) - end - - @impl true - def base_db_query() do - from rule in PageRule, - inner_join: s in assoc(rule, :site), - select: { - s.domain, - s.domain_changed_from, - %{struct(rule, ^@cached_schema_fields) | from_cache?: true} - } - end - - @impl true - def get_from_source(domain) do - query = - base_db_query() - |> where([..., site], site.domain == ^domain) - - case Plausible.Repo.one(query) do - {_, _, rule} -> %PageRule{rule | from_cache?: false} - _any -> nil - end - end - - @impl true - def unwrap_cache_keys(items) do - Enum.reduce(items, [], fn - {domain, nil, object}, acc -> - [{domain, object} | acc] - - {domain, domain_changed_from, object}, acc -> - [ - {domain, object}, - {domain_changed_from, object} | acc - ] - end) - end -end diff --git a/lib/plausible/shields.ex b/lib/plausible/shields.ex index 942b1fba0..25ba47d84 100644 --- a/lib/plausible/shields.ex +++ b/lib/plausible/shields.ex @@ -13,63 +13,11 @@ defmodule Plausible.Shields do @maximum_country_rules 30 def maximum_country_rules(), do: @maximum_country_rules - @maximum_page_rules 30 - def maximum_page_rules(), do: @maximum_page_rules - @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] def list_ip_rules(site_or_id) do list(Shield.IPRule, site_or_id) end - @spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean() - def ip_blocked?(%Site{domain: domain}, address) do - ip_blocked?(domain, address) - end - - def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do - case Shield.IPRuleCache.get({domain, address}) do - %Shield.IPRule{action: :deny} -> - true - - _ -> - false - end - end - - @spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean() - def page_blocked?(%Site{domain: domain}, address) do - page_blocked?(domain, address) - end - - def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do - page_rules = Shield.PageRuleCache.get(domain) - - if page_rules do - page_rules - |> List.wrap() - |> Enum.find_value(false, fn rule -> - rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname) - end) - else - false - end - end - - @spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean() - def country_blocked?(%Site{domain: domain}, country_code) do - country_blocked?(domain, country_code) - end - - def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do - case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do - %Shield.CountryRule{action: :deny} -> - true - - _ -> - false - end - end - @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} def add_ip_rule(site_or_id, params, opts \\ []) do @@ -111,28 +59,6 @@ defmodule Plausible.Shields do count(Shield.CountryRule, site_or_id) end - @spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()] - def list_page_rules(site_or_id) do - list(Shield.PageRule, site_or_id) - end - - @spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: - {:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()} - def add_page_rule(site_or_id, params, opts \\ []) do - opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules}) - add(Shield.PageRule, site_or_id, params, opts) - end - - @spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok - def remove_page_rule(site_or_id, rule_id) do - remove(Shield.PageRule, site_or_id, rule_id) - end - - @spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer() - def count_page_rules(site_or_id) do - count(Shield.PageRule, site_or_id) - end - defp list(schema, %Site{id: id}) do list(schema, id) end diff --git a/lib/plausible/shields/country_test.exs b/lib/plausible/shields/country_test.exs deleted file mode 100644 index 8b2218688..000000000 --- a/lib/plausible/shields/country_test.exs +++ /dev/null @@ -1,136 +0,0 @@ -defmodule Plausible.Shields.CountryTest do - use Plausible.DataCase - import Plausible.Shields - - setup do - site = insert(:site) - {:ok, %{site: site}} - end - - describe "add_country_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{}) - assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported country", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) - assert changeset.errors == [country_code: {"is invalid", []}] - refute changeset.valid? - end - - test "incorrect country format", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) - - assert changeset.errors == - [ - {:country_code, {"is invalid", []}}, - {:country_code, - {"should be %{count} character(s)", - [count: 2, validation: :length, kind: :is, type: :string]}} - ] - - refute changeset.valid? - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) - refute changeset.valid? - - assert changeset.errors == [ - country_code: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_country_site_id_country_code_index"} - ]} - ] - end - - test "over limit", %{site: site} do - country_codes = - Location.Country.all() - |> Enum.take(Plausible.Shields.maximum_country_rules()) - |> Enum.map(& &1.alpha_2) - - for cc <- country_codes do - assert {:ok, _} = - add_country_rule(site, %{"country_code" => cc}) - end - - assert count_country_rules(site) == maximum_country_rules() - - assert {:error, changeset} = - add_country_rule(site, %{"country_code" => "US"}) - - refute changeset.valid? - assert changeset.errors == [country_code: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_country_rule(site, %{"country_code" => "EE"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_country_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) - assert remove_country_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.CountryRule, rule.id) - assert remove_country_rule(site, rule.id) == :ok - end - end - - describe "list_country_rules/1" do - test "empty", %{site: site} do - assert(list_country_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) - :timer.sleep(1000) - {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) - assert [^r2, ^r1] = list_country_rules(site) - end - end - - describe "count_country_rules/1" do - test "counts", %{site: site} do - assert count_country_rules(site) == 0 - {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert count_country_rules(site) == 1 - {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) - assert count_country_rules(site) == 2 - end - end - - describe "Country Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_country_rules(site.id) == 0 - assert list_country_rules(site.id) == [] - - assert {:ok, rule} = - add_country_rule(site.id, %{"country_code" => "EE"}) - - add_country_rule(site2, %{"country_code" => "PL"}) - - assert count_country_rules(site) == 1 - assert [^rule] = list_country_rules(site) - assert rule.country_code == "EE" - assert rule.action == :deny - refute rule.from_cache? - assert country_blocked?(site, "ee") - refute country_blocked?(site, "xx") - end - end -end diff --git a/lib/plausible/shields/ip_test.exs b/lib/plausible/shields/ip_test.exs deleted file mode 100644 index bd597a392..000000000 --- a/lib/plausible/shields/ip_test.exs +++ /dev/null @@ -1,147 +0,0 @@ -defmodule Plausible.Shields.IPTest do - use Plausible.DataCase - import Plausible.Shields - - setup do - site = insert(:site) - {:ok, %{site: site}} - end - - describe "add_ip_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{}) - assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported netmask", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) - assert changeset.errors == [inet: {"netmask unsupported", []}] - refute changeset.valid? - assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) - end - - test "incorrect ip", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) - - assert changeset.errors == [ - inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} - ] - - refute changeset.valid? - end - - test "non-strict IPs", %{site: site} do - assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - refute changeset.valid? - - assert changeset.errors == [ - inet: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_ip_site_id_inet_index"} - ]} - ] - end - - test "ipv6", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "ipv4", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "over limit", %{site: site} do - for i <- 1..maximum_ip_rules() do - assert {:ok, _} = - add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) - end - - assert count_ip_rules(site) == maximum_ip_rules() - - assert {:error, changeset} = - add_ip_rule(site, %{"inet" => "1.1.1.31"}) - - refute changeset.valid? - assert changeset.errors == [inet: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_ip_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert remove_ip_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.IPRule, rule.id) - assert remove_ip_rule(site, rule.id) == :ok - end - end - - describe "list_ip_rules/1" do - test "empty", %{site: site} do - assert(list_ip_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - :timer.sleep(1000) - {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert [^r2, ^r1] = list_ip_rules(site) - end - end - - describe "count_ip_rules/1" do - test "counts", %{site: site} do - assert count_ip_rules(site) == 0 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert count_ip_rules(site) == 1 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert count_ip_rules(site) == 2 - end - end - - describe "IP Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_ip_rules(site.id) == 0 - assert list_ip_rules(site.id) == [] - - assert {:ok, rule} = - add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - assert count_ip_rules(site) == 1 - assert [^rule] = list_ip_rules(site) - assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} - assert rule.description == "Localhost" - assert rule.action == :deny - refute rule.from_cache? - assert ip_blocked?(site, "127.0.0.1") - refute ip_blocked?(site, "127.0.0.2") - end - end -end diff --git a/lib/plausible/shields/page_test.exs b/lib/plausible/shields/page_test.exs deleted file mode 100644 index 6134e9903..000000000 --- a/lib/plausible/shields/page_test.exs +++ /dev/null @@ -1,174 +0,0 @@ -defmodule Plausible.Shields.PageTest do - use Plausible.DataCase - import Plausible.Shields - - setup do - site = insert(:site) - {:ok, %{site: site}} - end - - describe "add_page_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_page_rule(site, %{}) - assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "no slash", %{site: site} do - assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"}) - assert changeset.errors == [page_path: {"must start with /", []}] - refute changeset.valid? - end - - test "lengthy", %{site: site} do - long = "/" <> :binary.copy("a", 251) - assert {:error, changeset} = add_page_rule(site, %{"page_path" => long}) - assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors - refute changeset.valid? - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) - assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"}) - refute changeset.valid? - - assert changeset.errors == [ - page_path: - {"rule already exists", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} - ]} - ] - end - - test "equivalent rules are counted as dupes", %{site: site} do - assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) - assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"}) - - assert changeset.errors == [ - page_path: - {"rule already exists", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} - ]} - ] - end - - test "regex storage: wildcard", %{site: site} do - assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"}) - assert rule.page_path_pattern == "^/test/.*$" - end - - test "regex storage: no wildcard", %{site: site} do - assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) - assert rule.page_path_pattern == "^/test$" - end - - test "regex storage: escaping", %{site: site} do - assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"}) - assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$" - end - - test "over limit", %{site: site} do - for i <- 1..maximum_page_rules() do - assert {:ok, _} = - add_page_rule(site, %{"page_path" => "/test/#{i}"}) - end - - assert count_page_rules(site) == maximum_page_rules() - - assert {:error, changeset} = - add_page_rule(site, %{"page_path" => "/test/31"}) - - refute changeset.valid? - assert changeset.errors == [page_path: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_page_rule(site, %{"page_path" => "/test"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "page pattern matching" do - test "no wildcard", %{site: site} do - assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) - assert page_blocked?(site, "/test") - refute page_blocked?(site, "/test/hello") - refute page_blocked?(site, "test") - end - - test "wildcard", %{site: site} do - assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) - refute page_blocked?(site, "/test") - assert page_blocked?(site, "/test/") - assert page_blocked?(site, "/test/hello") - refute page_blocked?(site, "test") - refute page_blocked?(site, "/testing") - end - end - - describe "remove_page_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) - assert remove_page_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.PageRule, rule.id) - assert remove_page_rule(site, rule.id) == :ok - end - end - - describe "list_page_rules/1" do - test "empty", %{site: site} do - assert(list_page_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_page_rule(site, %{"page_path" => "/test"}) - :timer.sleep(1000) - {:ok, r2} = add_page_rule(site, %{"page_path" => "/test"}) - assert [^r2, ^r1] = list_page_rules(site) - end - end - - describe "count_page_rules/1" do - test "counts", %{site: site} do - assert count_page_rules(site) == 0 - {:ok, _} = add_page_rule(site, %{"page_path" => "/test1"}) - assert count_page_rules(site) == 1 - {:ok, _} = add_page_rule(site, %{"page_path" => "/test2"}) - assert count_page_rules(site) == 2 - end - end - - describe "Page Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_page_rules(site.id) == 0 - assert list_page_rules(site.id) == [] - - assert {:ok, rule} = - add_page_rule(site.id, %{ - "page_path" => "/test" - }) - - add_page_rule(site2, %{"page_path" => "/test"}) - - assert count_page_rules(site) == 1 - assert [%{id: rule_id}] = list_page_rules(site) - assert rule.id == rule_id - assert rule.page_path == "/test" - assert rule.action == :deny - refute rule.from_cache? - assert page_blocked?(site, "/test") - refute page_blocked?(site, "/testing") - end - end -end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 77c9a9343..436c391c7 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -257,7 +257,7 @@ defmodule PlausibleWeb.SiteController do end def settings_shields(conn, %{"shield" => shield}) - when shield in ["ip_addresses", "countries", "pages"] do + when shield in ["ip_addresses", "countries"] do site = conn.assigns.site conn diff --git a/lib/plausible_web/live/shields/page_rules.ex b/lib/plausible_web/live/shields/page_rules.ex deleted file mode 100644 index 3ff38e12b..000000000 --- a/lib/plausible_web/live/shields/page_rules.ex +++ /dev/null @@ -1,304 +0,0 @@ -defmodule PlausibleWeb.Live.Shields.PageRules do - @moduledoc """ - LiveView allowing page Rules management - """ - - use Phoenix.LiveComponent, global_prefixes: ~w(x-) - use Phoenix.HTML - - alias PlausibleWeb.Live.Components.Modal - alias Plausible.Shields - alias Plausible.Shield - - import PlausibleWeb.ErrorHelpers - - def update(assigns, socket) do - socket = - socket - |> assign( - page_rules_count: assigns[:page_rules_count] || socket.assigns.page_rules_count, - site: assigns[:site] || socket.assigns.site, - current_user: assigns[:current_user] || socket.assigns.current_user, - form: new_form() - ) - |> assign_new(:page_rules, fn %{site: site} -> - Shields.list_page_rules(site) - end) - |> assign_new(:redundant_rules, fn %{page_rules: page_rules} -> - detect_redundancy(page_rules) - end) - - {:ok, socket} - end - - def render(assigns) do - ~H""" -
-
-
-

- Pages Block List -

-

- Reject incoming traffic for specific pages -

- - -
-
-
- - + Add Page - -
- = Shields.maximum_page_rules()} - class="mt-4" - title="Maximum number of pages reached" - > -

- You've reached the maximum number of pages you can block (<%= Shields.maximum_page_rules() %>). Please remove one before adding another. -

-
-
- - <.live_component module={Modal} id="page-rule-form-modal"> - <.form - :let={f} - for={@form} - phx-submit="save-page-rule" - phx-target={@myself} - class="max-w-md w-full mx-auto bg-white dark:bg-gray-800" - > -

Add Page to Block List

- - <.live_component - submit_name="page_rule[page_path]" - submit_value={f[:page_path].value} - display_value={f[:page_path].value || ""} - module={PlausibleWeb.Live.Components.ComboBox} - suggest_fun={fn input, options -> suggest_page_paths(input, options, @site) end} - id={f[:page_path].id} - creatable - /> - - <%= error_tag(f, :page_path) %> - -

- You can use a wildcard (*) to match multiple pages. For example, - /blog/* - will match /blog/post. - Once added, we will start rejecting traffic from this page within a few minutes. -

-
- - Add Page → - -
- - - -

- No Page Rules configured for this Site. -

-
- - - - - - - - - - <%= for rule <- @page_rules do %> - - - - - - - <% end %> - -
- page - - Status - - Remove -
-
- - <%= rule.page_path %> - -
-
-
- - Blocked - - - Allowed - - - - - -
-
- -
-
-
-
- """ - end - - def handle_event("save-page-rule", %{"page_rule" => params}, socket) do - user = socket.assigns.current_user - - case Shields.add_page_rule( - socket.assigns.site.id, - params, - added_by: user - ) do - {:ok, rule} -> - page_rules = [rule | socket.assigns.page_rules] - - socket = - socket - |> Modal.close("page-rule-form-modal") - |> assign( - form: new_form(), - page_rules: page_rules, - page_rules_count: socket.assigns.page_rules_count + 1, - redundant_rules: detect_redundancy(page_rules) - ) - - # Make sure to clear the combobox input after adding a page rule, on subsequent modal reopening - send_update(PlausibleWeb.Live.Components.ComboBox, - id: "page_rule_page_code", - display_value: "" - ) - - send_flash( - :success, - "Page rule added successfully. Traffic will be rejected within a few minutes." - ) - - {:noreply, socket} - - {:error, %Ecto.Changeset{} = changeset} -> - {:noreply, assign(socket, form: to_form(changeset))} - end - end - - def handle_event("remove-page-rule", %{"rule-id" => rule_id}, socket) do - Shields.remove_page_rule(socket.assigns.site.id, rule_id) - - send_flash( - :success, - "Page rule removed successfully. Traffic will be resumed within a few minutes." - ) - - page_rules = Enum.reject(socket.assigns.page_rules, &(&1.id == rule_id)) - - {:noreply, - socket - |> assign( - page_rules_count: socket.assigns.page_rules_count - 1, - page_rules: page_rules, - redundant_rules: detect_redundancy(page_rules) - )} - end - - def send_flash(kind, message) do - send(self(), {:flash, kind, message}) - end - - defp new_form() do - %Shield.PageRule{} - |> Shield.PageRule.changeset(%{}) - |> to_form() - end - - defp format_added_at(dt, tz) do - dt - |> Plausible.Timezones.to_datetime_in_timezone(tz) - |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") - end - - def suggest_page_paths(input, _options, site) do - query = Plausible.Stats.Query.from(site, %{}) - - site - |> Plausible.Stats.filter_suggestions(query, "page", input) - |> Enum.map(fn %{label: label, value: value} -> {label, value} end) - end - - defp detect_redundancy(page_rules) do - page_rules - |> Enum.reduce(%{}, fn rule, acc -> - {[^rule], remaining_rules} = - Enum.split_with( - page_rules, - fn r -> r == rule end - ) - - conflicting = - remaining_rules - |> Enum.filter(fn candidate -> - rule - |> Map.fetch!(:page_path_pattern) - |> maybe_compile() - |> Regex.match?(candidate.page_path) - end) - |> Enum.map(& &1.id) - - Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc -> - Map.update(acc, conflicting_rule_id, [rule.page_path], fn existing -> - [rule.page_path | existing] - end) - end) - end) - end - - defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern) - defp maybe_compile(%Regex{} = pattern), do: pattern -end diff --git a/lib/plausible_web/live/shields/pages.ex b/lib/plausible_web/live/shields/pages.ex deleted file mode 100644 index cd42c47b3..000000000 --- a/lib/plausible_web/live/shields/pages.ex +++ /dev/null @@ -1,53 +0,0 @@ -defmodule PlausibleWeb.Live.Shields.Pages do - @moduledoc """ - LiveView for IP Addresses Shield - """ - use PlausibleWeb, :live_view - use Phoenix.HTML - - alias Plausible.Shields - alias Plausible.Sites - - def mount( - _params, - %{ - "domain" => domain, - "current_user_id" => user_id - }, - socket - ) do - socket = - socket - |> assign_new(:site, fn -> - Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin]) - end) - |> assign_new(:page_rules_count, fn %{site: site} -> - Shields.count_page_rules(site) - end) - |> assign_new(:current_user, fn -> - Plausible.Repo.get(Plausible.Auth.User, user_id) - end) - - {:ok, socket} - end - - def render(assigns) do - ~H""" -
- <.flash_messages flash={@flash} /> - <.live_component - module={PlausibleWeb.Live.Shields.PageRules} - current_user={@current_user} - page_rules_count={@page_rules_count} - site={@site} - id="page-rules-#{@current_user.id}" - /> -
- """ - end - - def handle_info({:flash, kind, message}, socket) do - socket = put_live_flash(socket, kind, message) - {:noreply, socket} - end -end diff --git a/lib/plausible_web/templates/site/settings_shields.html.heex b/lib/plausible_web/templates/site/settings_shields.html.heex index 489d49daf..411c39fd3 100644 --- a/lib/plausible_web/templates/site/settings_shields.html.heex +++ b/lib/plausible_web/templates/site/settings_shields.html.heex @@ -14,11 +14,4 @@ "domain" => @site.domain } ) %> - <% "pages" -> %> - <%= live_render(@conn, PlausibleWeb.Live.Shields.Pages, - session: %{ - "site_id" => @site.id, - "domain" => @site.domain - } - ) %> <% end %> diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index 42923e6f2..0b587379d 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -65,15 +65,10 @@ defmodule PlausibleWeb.LayoutView do %{ key: "Shields", icon: :shield_exclamation, - value: - [ - %{key: "IP Addresses", value: "shields/ip_addresses"}, - %{key: "Countries", value: "shields/countries"}, - if FunWithFlags.enabled?(:shield_pages) do - %{key: "Pages", value: "shields/pages"} - end - ] - |> Enum.reject(&is_nil/1) + value: [ + %{key: "IP Addresses", value: "shields/ip_addresses"}, + %{key: "Countries", value: "shields/countries"} + ] }, %{key: "Email Reports", value: "email-reports", icon: :envelope}, if conn.assigns[:current_user_role] == :owner do diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index da7b842d9..e1b499f76 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -11,7 +11,6 @@ # and so on) as they will fail if something goes wrong. FunWithFlags.enable(:imports_exports) -FunWithFlags.enable(:shield_pages) user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 8714e0848..b6627d8bc 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -158,25 +158,6 @@ defmodule Plausible.Ingestion.EventTest do assert dropped.drop_reason == :site_country_blocklist end - test "event pipeline drops a request when page is on blocklist" do - site = insert(:site) - - payload = %{ - name: "pageview", - url: "http://dummy.site/blocked/page", - domain: site.domain - } - - conn = build_conn(:post, "/api/events", payload) - - {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/blocked/**"}) - - assert {:ok, request} = Request.build(conn) - - assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) - assert dropped.drop_reason == :site_page_blocklist - end - test "event pipeline drops events for site with accept_trafic_until in the past" do yesterday = Date.add(Date.utc_today(), -1) diff --git a/test/plausible/shield/page_rule_cache_test.exs b/test/plausible/shield/page_rule_cache_test.exs deleted file mode 100644 index ba26f3cf9..000000000 --- a/test/plausible/shield/page_rule_cache_test.exs +++ /dev/null @@ -1,107 +0,0 @@ -defmodule Plausible.Shield.PageRuleCacheTest do - use Plausible.DataCase, async: true - - alias Plausible.Shield.PageRule - alias Plausible.Shield.PageRuleCache - alias Plausible.Shields - - describe "public cache interface" do - test "cache caches page rules", %{test: test} do - cache_opts = [force?: true, cache_name: test] - - {:ok, _} = - Supervisor.start_link( - [ - {PageRuleCache, - [cache_name: test, child_id: :page_rules_cache_id, ets_options: [:bag]]} - ], - strategy: :one_for_one, - name: :"cache_supervisor_#{test}" - ) - - site1 = insert(:site) - site2 = insert(:site) - - {:ok, %{id: rid1}} = Shields.add_page_rule(site1, %{"page_path" => "/test/1"}) - {:ok, %{id: rid2}} = Shields.add_page_rule(site1, %{"page_path" => "/test/2"}) - {:ok, %{id: rid3}} = Shields.add_page_rule(site2, %{"page_path" => "/test/2"}) - - :ok = PageRuleCache.refresh_all(cache_name: test) - - :ok = Shields.remove_page_rule(site1, rid1) - - assert PageRuleCache.size(test) == 3 - - # the rule order should be deterministic, but with 1s timestamp sorting precision - # race conditions may happen during tests - assert rules = PageRuleCache.get(site1.domain, cache_opts) - rule_ids = Enum.map(rules, & &1.id) - assert rid1 in rule_ids - assert rid2 in rule_ids - assert length(rules) == 2 - - assert %PageRule{from_cache?: true, id: ^rid3} = - PageRuleCache.get(site2.domain, cache_opts) - - refute PageRuleCache.get("rogue.example.com", cache_opts) - end - - test "page path patterns are already compiled when fetched from cache", %{test: test} do - site = insert(:site) - - {:ok, _} = start_test_cache(test) - cache_opts = [force?: true, cache_name: test] - - {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/hello/**/world"}) - :ok = PageRuleCache.refresh_all(cache_name: test) - assert regex = PageRuleCache.get(site.domain, cache_opts).page_path_pattern - assert regex == ~r/^\/hello\/.*\/world$/ - end - - test "cache allows lookups for page paths on sites with changed domain", %{test: test} do - {:ok, _} = start_test_cache(test) - cache_opts = [force?: true, cache_name: test] - site = insert(:site, domain: "new.example.com", domain_changed_from: "old.example.com") - - {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/#{test}"}) - :ok = PageRuleCache.refresh_all(cache_name: test) - - assert PageRuleCache.get("old.example.com", cache_opts) - assert PageRuleCache.get("new.example.com", cache_opts) - refute PageRuleCache.get("rogue.example.com", cache_opts) - end - - test "refreshes only recently added pages rules", %{test: test} do - {:ok, _} = start_test_cache(test) - - domain = "site1.example.com" - site = insert(:site, domain: domain) - - cache_opts = [cache_name: test, force?: true] - - yesterday = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(-1 * 60 * 60 * 24) - |> NaiveDateTime.truncate(:second) - - {:ok, r1} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/1"}) - Ecto.Changeset.change(r1, inserted_at: yesterday, updated_at: yesterday) |> Repo.update!() - {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/2"}) - - assert PageRuleCache.get(domain, cache_opts) == nil - - assert :ok = PageRuleCache.refresh_updated_recently(cache_opts) - - assert %{page_path_pattern: ~r[^/test/2$]} = PageRuleCache.get(domain, cache_opts) - - assert :ok = PageRuleCache.refresh_all(cache_opts) - - assert [_, _] = PageRuleCache.get(domain, cache_opts) - end - end - - defp start_test_cache(cache_name) do - %{start: {m, f, a}} = PageRuleCache.child_spec(cache_name: cache_name, ets_options: [:bag]) - apply(m, f, a) - end -end diff --git a/test/plausible/shields_test.exs b/test/plausible/shields_test.exs index e69de29bb..c39da4587 100644 --- a/test/plausible/shields_test.exs +++ b/test/plausible/shields_test.exs @@ -0,0 +1,270 @@ +defmodule Plausible.ShieldsTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_ip_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{}) + assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported netmask", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) + assert changeset.errors == [inet: {"netmask unsupported", []}] + refute changeset.valid? + assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) + end + + test "incorrect ip", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) + + assert changeset.errors == [ + inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} + ] + + refute changeset.valid? + end + + test "non-strict IPs", %{site: site} do + assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + refute changeset.valid? + + assert changeset.errors == [ + inet: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_ip_site_id_inet_index"} + ]} + ] + end + + test "ipv6", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "ipv4", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "over limit", %{site: site} do + for i <- 1..maximum_ip_rules() do + assert {:ok, _} = + add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) + end + + assert count_ip_rules(site) == maximum_ip_rules() + + assert {:error, changeset} = + add_ip_rule(site, %{"inet" => "1.1.1.31"}) + + refute changeset.valid? + assert changeset.errors == [inet: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_ip_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert remove_ip_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.IPRule, rule.id) + assert remove_ip_rule(site, rule.id) == :ok + end + end + + describe "list_ip_rules/1" do + test "empty", %{site: site} do + assert(list_ip_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + :timer.sleep(1000) + {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert [^r2, ^r1] = list_ip_rules(site) + end + end + + describe "count_ip_rules/1" do + test "counts", %{site: site} do + assert count_ip_rules(site) == 0 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert count_ip_rules(site) == 1 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert count_ip_rules(site) == 2 + end + end + + describe "IP Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_ip_rules(site.id) == 0 + assert list_ip_rules(site.id) == [] + + assert {:ok, rule} = + add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + assert count_ip_rules(site) == 1 + assert [^rule] = list_ip_rules(site) + assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} + assert rule.description == "Localhost" + assert rule.action == :deny + refute rule.from_cache? + end + end + + describe "add_country_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{}) + assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported country", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) + assert changeset.errors == [country_code: {"is invalid", []}] + refute changeset.valid? + end + + test "incorrect country format", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) + + assert changeset.errors == + [ + {:country_code, {"is invalid", []}}, + {:country_code, + {"should be %{count} character(s)", + [count: 2, validation: :length, kind: :is, type: :string]}} + ] + + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) + refute changeset.valid? + + assert changeset.errors == [ + country_code: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_country_site_id_country_code_index"} + ]} + ] + end + + test "over limit", %{site: site} do + country_codes = + Location.Country.all() + |> Enum.take(Plausible.Shields.maximum_country_rules()) + |> Enum.map(& &1.alpha_2) + + for cc <- country_codes do + assert {:ok, _} = + add_country_rule(site, %{"country_code" => cc}) + end + + assert count_country_rules(site) == maximum_country_rules() + + assert {:error, changeset} = + add_country_rule(site, %{"country_code" => "US"}) + + refute changeset.valid? + assert changeset.errors == [country_code: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_country_rule(site, %{"country_code" => "EE"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_country_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) + assert remove_country_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.CountryRule, rule.id) + assert remove_country_rule(site, rule.id) == :ok + end + end + + describe "list_country_rules/1" do + test "empty", %{site: site} do + assert(list_country_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) + :timer.sleep(1000) + {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) + assert [^r2, ^r1] = list_country_rules(site) + end + end + + describe "count_country_rules/1" do + test "counts", %{site: site} do + assert count_country_rules(site) == 0 + {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert count_country_rules(site) == 1 + {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) + assert count_country_rules(site) == 2 + end + end + + describe "Country Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_country_rules(site.id) == 0 + assert list_country_rules(site.id) == [] + + assert {:ok, rule} = + add_country_rule(site.id, %{"country_code" => "EE"}) + + add_country_rule(site2, %{"country_code" => "PL"}) + + assert count_country_rules(site) == 1 + assert [^rule] = list_country_rules(site) + assert rule.country_code == "EE" + assert rule.action == :deny + refute rule.from_cache? + end + end +end diff --git a/test/plausible_web/live/shields/ip_addresses_test.exs b/test/plausible_web/live/shields/ip_addresses_test.exs index 16a4f2b1a..fd11a431d 100644 --- a/test/plausible_web/live/shields/ip_addresses_test.exs +++ b/test/plausible_web/live/shields/ip_addresses_test.exs @@ -95,7 +95,7 @@ defmodule PlausibleWeb.Live.Shields.IPAddressesTest do assert element_exists?(html, ~s/a[phx-click="prefill-own-ip-rule"]/) end - test "form modal does not contain link to add own IP if already added", %{ + test "form modal does not contains link to add own IP if already added", %{ site: site, conn: conn } do diff --git a/test/plausible_web/live/shields/pages_test.exs b/test/plausible_web/live/shields/pages_test.exs deleted file mode 100644 index 7f89456a8..000000000 --- a/test/plausible_web/live/shields/pages_test.exs +++ /dev/null @@ -1,176 +0,0 @@ -defmodule PlausibleWeb.Live.Shields.PagesTest do - use PlausibleWeb.ConnCase, async: true - - import Phoenix.LiveViewTest - import Plausible.Test.Support.HTML - - alias Plausible.Shields - - setup [:create_user, :create_site, :log_in] - - describe "Page Rules - static" do - test "renders page rules page with empty list", %{conn: conn, site: site} do - conn = get(conn, "/#{site.domain}/settings/shields/pages") - resp = html_response(conn, 200) - - assert resp =~ "No Page Rules configured for this Site" - assert resp =~ "Pages Block List" - end - - test "lists page rules with remove actions", %{conn: conn, site: site} do - {:ok, r1} = - Shields.add_page_rule(site, %{"page_path" => "/test/1"}) - - {:ok, r2} = - Shields.add_page_rule(site, %{"page_path" => "/test/2"}) - - conn = get(conn, "/#{site.domain}/settings/shields/pages") - resp = html_response(conn, 200) - - assert resp =~ "/test/1" - assert resp =~ "/test/2" - - assert remove_button_1 = find(resp, "#remove-page-rule-#{r1.id}") - assert remove_button_2 = find(resp, "#remove-page-rule-#{r2.id}") - - assert text_of_attr(remove_button_1, "phx-click" == "remove-page-rule") - assert text_of_attr(remove_button_1, "phx-value-rule-id" == r1.id) - assert text_of_attr(remove_button_2, "phx-click" == "remove-page-rule") - assert text_of_attr(remove_button_2, "phx-value-rule-id" == r2.id) - end - - test "add rule button is rendered", %{conn: conn, site: site} do - conn = get(conn, "/#{site.domain}/settings/shields/pages") - resp = html_response(conn, 200) - - assert element_exists?(resp, ~s/button#add-page-rule[x-data]/) - attr = text_of_attr(resp, ~s/button#add-page-rule/, "x-on:click") - - assert attr =~ "open-modal" - assert attr =~ "page-rule-form-modal" - end - - test "add rule button is not rendered when maximum reached", %{conn: conn, site: site} do - for i <- 1..Shields.maximum_page_rules() do - assert {:ok, _} = - Shields.add_page_rule(site, %{"page_path" => "/test/#{i}"}) - end - - conn = get(conn, "/#{site.domain}/settings/shields/pages") - resp = html_response(conn, 200) - - refute element_exists?(resp, ~s/button#add-page-rule[x-data]/) - assert resp =~ "Maximum number of pages reached" - assert resp =~ "You've reached the maximum number of pages you can block (30)" - end - end - - describe "Page Rules - LiveView" do - test "modal contains form", %{site: site, conn: conn} do - lv = get_liveview(conn, site) - html = render(lv) - - assert element_exists?( - html, - ~s/form[phx-submit="save-page-rule"] input[name="page_rule\[page_path\]"]/ - ) - - assert submit_button(html, ~s/form[phx-submit="save-page-rule"]/) - end - - test "submitting a valid Page saves it", %{conn: conn, site: site} do - lv = get_liveview(conn, site) - - lv - |> element("form") - |> render_submit(%{ - "page_rule[page_path]" => "/test/**" - }) - - html = render(lv) - - assert html =~ "/test/**" - - assert [%{page_path: "/test/**"}] = Shields.list_page_rules(site) - end - - test "submitting invalid Page renders error", %{conn: conn, site: site} do - lv = get_liveview(conn, site) - - lv - |> element("form") - |> render_submit(%{ - "page_rule[page_path]" => "WRONG" - }) - - html = render(lv) - assert html =~ "must start with /" - end - - test "clicking Remove deletes the rule", %{conn: conn, site: site} do - {:ok, _} = - Shields.add_page_rule(site, %{"page_path" => "/test/*/page"}) - - lv = get_liveview(conn, site) - - html = render(lv) - assert html =~ "/test/*/page" - - lv |> element(~s/button[phx-click="remove-page-rule"]/) |> render_click() - - html = render(lv) - refute html =~ "/test/*/page" - - assert Shields.count_page_rules(site) == 0 - end - - test "conclicting rules are annotated with a warning", %{conn: conn, site: site} do - lv = get_liveview(conn, site) - - lv - |> element("form") - |> render_submit(%{ - "page_rule[page_path]" => "/test/*" - }) - - html = render(lv) - refute html =~ "This rule might be redundant" - - lv - |> element("form") - |> render_submit(%{ - "page_rule[page_path]" => "/test/another" - }) - - html = render(lv) - - assert html =~ "/test/*" - assert html =~ "/test/another" - - assert html =~ - "This rule might be redundant because the following rules may match first:\n\n/test/*" - - broader_rule_id = - site - |> Shields.list_page_rules() - |> Enum.find(&(&1.page_path == "/test/*")) - |> Map.fetch!(:id) - - lv |> element(~s/button#remove-page-rule-#{broader_rule_id}/) |> render_click() - html = render(lv) - - assert html =~ "/test/another" - refute html =~ "/test/*" - - refute html =~ - "This rule might be redundant because the following rules may match first:\n\n/test/*" - end - - defp get_liveview(conn, site) do - conn = assign(conn, :live_module, PlausibleWeb.Live.Shields) - {:ok, lv, _html} = live(conn, "/#{site.domain}/settings/shields/pages") - - lv - end - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 78171693a..75e431245 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -3,7 +3,6 @@ Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) Application.ensure_all_started(:double) FunWithFlags.enable(:window_time_on_page) FunWithFlags.enable(:imports_exports) -FunWithFlags.enable(:shield_pages) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) if Mix.env() == :small_test do From 2fae0146a46abdea77e6d194972e978f1a248155 Mon Sep 17 00:00:00 2001 From: hq1 Date: Mon, 25 Mar 2024 10:36:22 +0100 Subject: [PATCH 3/9] Reapply 3918 (#3940) * Reapply "Pages shield (#3918)" This reverts commit 33b5c106547072fde782fdeec7828f067e2ceb12. * Make the FF check work against the site actor --- CHANGELOG.md | 2 + lib/plausible/application.ex | 15 + lib/plausible/ingestion/event.ex | 31 +- lib/plausible/shield/country_rule.ex | 2 +- lib/plausible/shield/page_rule.ex | 85 +++++ lib/plausible/shield/page_rule_cache.ex | 65 ++++ lib/plausible/shields.ex | 74 +++++ lib/plausible/shields/country_test.exs | 136 ++++++++ lib/plausible/shields/ip_test.exs | 147 +++++++++ lib/plausible/shields/page_test.exs | 174 ++++++++++ .../controllers/site_controller.ex | 2 +- lib/plausible_web/live/shields/page_rules.ex | 304 ++++++++++++++++++ lib/plausible_web/live/shields/pages.ex | 53 +++ .../templates/site/settings_shields.html.heex | 7 + lib/plausible_web/views/layout_view.ex | 13 +- priv/repo/seeds.exs | 1 + test/plausible/ingestion/event_test.exs | 19 ++ .../plausible/shield/page_rule_cache_test.exs | 107 ++++++ test/plausible/shields_test.exs | 270 ---------------- .../live/shields/ip_addresses_test.exs | 2 +- .../plausible_web/live/shields/pages_test.exs | 176 ++++++++++ test/test_helper.exs | 1 + 22 files changed, 1395 insertions(+), 291 deletions(-) create mode 100644 lib/plausible/shield/page_rule.ex create mode 100644 lib/plausible/shield/page_rule_cache.ex create mode 100644 lib/plausible/shields/country_test.exs create mode 100644 lib/plausible/shields/ip_test.exs create mode 100644 lib/plausible/shields/page_test.exs create mode 100644 lib/plausible_web/live/shields/page_rules.ex create mode 100644 lib/plausible_web/live/shields/pages.ex create mode 100644 test/plausible/shield/page_rule_cache_test.exs create mode 100644 test/plausible_web/live/shields/pages_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ed59f16c..a9263eb8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ### Added +- Pages Block List in Site Settings +- Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export - Ability to display total conversions (with a goal filter) on the main graph - Add `conversion_rate` to Stats API Timeseries and on the main graph diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 3f8117f42..f197ec6a0 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -77,6 +77,21 @@ defmodule Plausible.Application do interval: :timer.seconds(35), warmer_fn: :refresh_updated_recently ]}, + {Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.PageRuleCache.All, + cache_impl: Plausible.Shield.PageRuleCache, + interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)), + warmer_fn: :refresh_all + ]}, + {Plausible.Cache.Warmer, + [ + child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated, + cache_impl: Plausible.Shield.PageRuleCache, + interval: :timer.seconds(35), + warmer_fn: :refresh_updated_recently + ]}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, PlausibleWeb.Endpoint, {Oban, Application.get_env(:plausible, Oban)}, diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 8c66c01d7..6dd680f61 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -29,6 +29,7 @@ defmodule Plausible.Ingestion.Event do | :dc_ip | :site_ip_blocklist | :site_country_blocklist + | :site_page_blocklist @type t() :: %__MODULE__{ domain: String.t() | nil, @@ -120,6 +121,7 @@ defmodule Plausible.Ingestion.Event do [ &drop_datacenter_ip/1, &drop_shield_rule_ip/1, + &drop_shield_rule_page/1, &put_geolocation/1, &drop_shield_rule_country/1, &put_user_agent/1, @@ -182,15 +184,18 @@ defmodule Plausible.Ingestion.Event do end defp drop_shield_rule_ip(%__MODULE__{} = event) do - domain = event.domain - address = event.request.remote_ip - - case Plausible.Shield.IPRuleCache.get({domain, address}) do - %Plausible.Shield.IPRule{action: :deny} -> - drop(event, :site_ip_blocklist) + if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do + drop(event, :site_ip_blocklist) + else + event + end + end - _ -> - event + defp drop_shield_rule_page(%__MODULE__{} = event) do + if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do + drop(event, :site_page_blocklist) + else + event end end @@ -263,12 +268,10 @@ defmodule Plausible.Ingestion.Event do %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event ) when is_binary(domain) and is_binary(cc) do - case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do - %Plausible.Shield.CountryRule{action: :deny} -> - drop(event, :site_country_blocklist) - - _ -> - event + if Plausible.Shields.country_blocked?(domain, cc) do + drop(event, :site_country_blocklist) + else + event end end diff --git a/lib/plausible/shield/country_rule.ex b/lib/plausible/shield/country_rule.ex index 85bc73456..067928f34 100644 --- a/lib/plausible/shield/country_rule.ex +++ b/lib/plausible/shield/country_rule.ex @@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do |> cast(attrs, [:site_id, :country_code]) |> validate_required([:site_id, :country_code]) |> validate_length(:country_code, is: 2) - |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc -> + |> validate_change(:country_code, fn :country_code, cc -> if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do [] else diff --git a/lib/plausible/shield/page_rule.ex b/lib/plausible/shield/page_rule.ex new file mode 100644 index 000000000..e874e185c --- /dev/null +++ b/lib/plausible/shield/page_rule.ex @@ -0,0 +1,85 @@ +defmodule CompiledRegex do + @moduledoc """ + Ensures that the regex is compiled on load + """ + use Ecto.Type + + def type, do: :string + + def cast(val) when is_binary(val), do: {:ok, val} + def cast(_), do: :error + + def load(val), do: {:ok, Regex.compile!(val)} + def dump(val), do: {:ok, val} +end + +defmodule Plausible.Shield.PageRule do + @moduledoc """ + Schema for Pages block list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_page" do + belongs_to :site, Plausible.Site + field :page_path, :string + field :page_path_pattern, CompiledRegex + field :action, Ecto.Enum, values: [:deny, :allow], default: :deny + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule \\ %__MODULE__{}, attrs) do + rule + |> cast(attrs, [:site_id, :page_path]) + |> validate_required([:site_id, :page_path]) + |> validate_length(:page_path, max: 250) + |> validate_change(:page_path, fn :page_path, p -> + if not String.starts_with?(p, "/") do + [page_path: "must start with /"] + else + [] + end + end) + |> store_regex() + |> unique_constraint(:page_path_pattern, + name: :shield_rules_page_site_id_page_path_pattern_index, + error_key: :page_path, + message: "rule already exists" + ) + end + + defp store_regex(changeset) do + case get_field(changeset, :page_path) do + "/" <> _ = page_path -> + regex = + page_path + |> Regex.escape() + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + regex = "^#{regex}$" + + verify_valid_regex(changeset, regex) + + _ -> + changeset + end + end + + defp verify_valid_regex(changeset, regex) do + case Regex.compile(regex) do + {:ok, _} -> + put_change(changeset, :page_path_pattern, regex) + + {:error, _} -> + add_error(changeset, :page_path, "could not compile regular expression") + end + end +end diff --git a/lib/plausible/shield/page_rule_cache.ex b/lib/plausible/shield/page_rule_cache.ex new file mode 100644 index 000000000..e9ba3039d --- /dev/null +++ b/lib/plausible/shield/page_rule_cache.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Shield.PageRuleCache do + @moduledoc """ + Allows retrieving Page Rules by domain + """ + alias Plausible.Shield.PageRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :page_blocklist_by_domain + + @cached_schema_fields ~w( + id + page_path_pattern + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_page_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(PageRule, :count) + end + + @impl true + def base_db_query() do + from rule in PageRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source(domain) do + query = + base_db_query() + |> where([..., site], site.domain == ^domain) + + case Plausible.Repo.one(query) do + {_, _, rule} -> %PageRule{rule | from_cache?: false} + _any -> nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {domain, object}, + {domain_changed_from, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shields.ex b/lib/plausible/shields.ex index 25ba47d84..942b1fba0 100644 --- a/lib/plausible/shields.ex +++ b/lib/plausible/shields.ex @@ -13,11 +13,63 @@ defmodule Plausible.Shields do @maximum_country_rules 30 def maximum_country_rules(), do: @maximum_country_rules + @maximum_page_rules 30 + def maximum_page_rules(), do: @maximum_page_rules + @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] def list_ip_rules(site_or_id) do list(Shield.IPRule, site_or_id) end + @spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def ip_blocked?(%Site{domain: domain}, address) do + ip_blocked?(domain, address) + end + + def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do + case Shield.IPRuleCache.get({domain, address}) do + %Shield.IPRule{action: :deny} -> + true + + _ -> + false + end + end + + @spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def page_blocked?(%Site{domain: domain}, address) do + page_blocked?(domain, address) + end + + def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do + page_rules = Shield.PageRuleCache.get(domain) + + if page_rules do + page_rules + |> List.wrap() + |> Enum.find_value(false, fn rule -> + rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname) + end) + else + false + end + end + + @spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def country_blocked?(%Site{domain: domain}, country_code) do + country_blocked?(domain, country_code) + end + + def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do + case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do + %Shield.CountryRule{action: :deny} -> + true + + _ -> + false + end + end + @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} def add_ip_rule(site_or_id, params, opts \\ []) do @@ -59,6 +111,28 @@ defmodule Plausible.Shields do count(Shield.CountryRule, site_or_id) end + @spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()] + def list_page_rules(site_or_id) do + list(Shield.PageRule, site_or_id) + end + + @spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()} + def add_page_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules}) + add(Shield.PageRule, site_or_id, params, opts) + end + + @spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_page_rule(site_or_id, rule_id) do + remove(Shield.PageRule, site_or_id, rule_id) + end + + @spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_page_rules(site_or_id) do + count(Shield.PageRule, site_or_id) + end + defp list(schema, %Site{id: id}) do list(schema, id) end diff --git a/lib/plausible/shields/country_test.exs b/lib/plausible/shields/country_test.exs new file mode 100644 index 000000000..8b2218688 --- /dev/null +++ b/lib/plausible/shields/country_test.exs @@ -0,0 +1,136 @@ +defmodule Plausible.Shields.CountryTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_country_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{}) + assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported country", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) + assert changeset.errors == [country_code: {"is invalid", []}] + refute changeset.valid? + end + + test "incorrect country format", %{site: site} do + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) + + assert changeset.errors == + [ + {:country_code, {"is invalid", []}}, + {:country_code, + {"should be %{count} character(s)", + [count: 2, validation: :length, kind: :is, type: :string]}} + ] + + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) + refute changeset.valid? + + assert changeset.errors == [ + country_code: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_country_site_id_country_code_index"} + ]} + ] + end + + test "over limit", %{site: site} do + country_codes = + Location.Country.all() + |> Enum.take(Plausible.Shields.maximum_country_rules()) + |> Enum.map(& &1.alpha_2) + + for cc <- country_codes do + assert {:ok, _} = + add_country_rule(site, %{"country_code" => cc}) + end + + assert count_country_rules(site) == maximum_country_rules() + + assert {:error, changeset} = + add_country_rule(site, %{"country_code" => "US"}) + + refute changeset.valid? + assert changeset.errors == [country_code: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_country_rule(site, %{"country_code" => "EE"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_country_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) + assert remove_country_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.CountryRule, rule.id) + assert remove_country_rule(site, rule.id) == :ok + end + end + + describe "list_country_rules/1" do + test "empty", %{site: site} do + assert(list_country_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) + :timer.sleep(1000) + {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) + assert [^r2, ^r1] = list_country_rules(site) + end + end + + describe "count_country_rules/1" do + test "counts", %{site: site} do + assert count_country_rules(site) == 0 + {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) + assert count_country_rules(site) == 1 + {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) + assert count_country_rules(site) == 2 + end + end + + describe "Country Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_country_rules(site.id) == 0 + assert list_country_rules(site.id) == [] + + assert {:ok, rule} = + add_country_rule(site.id, %{"country_code" => "EE"}) + + add_country_rule(site2, %{"country_code" => "PL"}) + + assert count_country_rules(site) == 1 + assert [^rule] = list_country_rules(site) + assert rule.country_code == "EE" + assert rule.action == :deny + refute rule.from_cache? + assert country_blocked?(site, "ee") + refute country_blocked?(site, "xx") + end + end +end diff --git a/lib/plausible/shields/ip_test.exs b/lib/plausible/shields/ip_test.exs new file mode 100644 index 000000000..bd597a392 --- /dev/null +++ b/lib/plausible/shields/ip_test.exs @@ -0,0 +1,147 @@ +defmodule Plausible.Shields.IPTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_ip_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{}) + assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "unsupported netmask", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) + assert changeset.errors == [inet: {"netmask unsupported", []}] + refute changeset.valid? + assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) + end + + test "incorrect ip", %{site: site} do + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) + + assert changeset.errors == [ + inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} + ] + + refute changeset.valid? + end + + test "non-strict IPs", %{site: site} do + assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) + refute changeset.valid? + + assert changeset.errors == [ + inet: + {"has already been taken", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_ip_site_id_inet_index"} + ]} + ] + end + + test "ipv6", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "ipv4", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}) + + assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) + end + + test "over limit", %{site: site} do + for i <- 1..maximum_ip_rules() do + assert {:ok, _} = + add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) + end + + assert count_ip_rules(site) == maximum_ip_rules() + + assert {:error, changeset} = + add_ip_rule(site, %{"inet" => "1.1.1.31"}) + + refute changeset.valid? + assert changeset.errors == [inet: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_ip_rule(site, %{"inet" => "1.1.1.1"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "remove_ip_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert remove_ip_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.IPRule, rule.id) + assert remove_ip_rule(site, rule.id) == :ok + end + end + + describe "list_ip_rules/1" do + test "empty", %{site: site} do + assert(list_ip_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + :timer.sleep(1000) + {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert [^r2, ^r1] = list_ip_rules(site) + end + end + + describe "count_ip_rules/1" do + test "counts", %{site: site} do + assert count_ip_rules(site) == 0 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) + assert count_ip_rules(site) == 1 + {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) + assert count_ip_rules(site) == 2 + end + end + + describe "IP Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_ip_rules(site.id) == 0 + assert list_ip_rules(site.id) == [] + + assert {:ok, rule} = + add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) + + assert count_ip_rules(site) == 1 + assert [^rule] = list_ip_rules(site) + assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} + assert rule.description == "Localhost" + assert rule.action == :deny + refute rule.from_cache? + assert ip_blocked?(site, "127.0.0.1") + refute ip_blocked?(site, "127.0.0.2") + end + end +end diff --git a/lib/plausible/shields/page_test.exs b/lib/plausible/shields/page_test.exs new file mode 100644 index 000000000..6134e9903 --- /dev/null +++ b/lib/plausible/shields/page_test.exs @@ -0,0 +1,174 @@ +defmodule Plausible.Shields.PageTest do + use Plausible.DataCase + import Plausible.Shields + + setup do + site = insert(:site) + {:ok, %{site: site}} + end + + describe "add_page_rule/2" do + test "no input", %{site: site} do + assert {:error, changeset} = add_page_rule(site, %{}) + assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}] + refute changeset.valid? + end + + test "no slash", %{site: site} do + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"}) + assert changeset.errors == [page_path: {"must start with /", []}] + refute changeset.valid? + end + + test "lengthy", %{site: site} do + long = "/" <> :binary.copy("a", 251) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => long}) + assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors + refute changeset.valid? + end + + test "double insert", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"}) + refute changeset.valid? + + assert changeset.errors == [ + page_path: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} + ]} + ] + end + + test "equivalent rules are counted as dupes", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) + assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"}) + + assert changeset.errors == [ + page_path: + {"rule already exists", + [ + {:constraint, :unique}, + {:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"} + ]} + ] + end + + test "regex storage: wildcard", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"}) + assert rule.page_path_pattern == "^/test/.*$" + end + + test "regex storage: no wildcard", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) + assert rule.page_path_pattern == "^/test$" + end + + test "regex storage: escaping", %{site: site} do + assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"}) + assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$" + end + + test "over limit", %{site: site} do + for i <- 1..maximum_page_rules() do + assert {:ok, _} = + add_page_rule(site, %{"page_path" => "/test/#{i}"}) + end + + assert count_page_rules(site) == maximum_page_rules() + + assert {:error, changeset} = + add_page_rule(site, %{"page_path" => "/test/31"}) + + refute changeset.valid? + assert changeset.errors == [page_path: {"maximum reached", []}] + end + + test "with added_by", %{site: site} do + assert {:ok, rule} = + add_page_rule(site, %{"page_path" => "/test"}, + added_by: build(:user, name: "Joe", email: "joe@example.com") + ) + + assert rule.added_by == "Joe " + end + end + + describe "page pattern matching" do + test "no wildcard", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"}) + assert page_blocked?(site, "/test") + refute page_blocked?(site, "/test/hello") + refute page_blocked?(site, "test") + end + + test "wildcard", %{site: site} do + assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"}) + refute page_blocked?(site, "/test") + assert page_blocked?(site, "/test/") + assert page_blocked?(site, "/test/hello") + refute page_blocked?(site, "test") + refute page_blocked?(site, "/testing") + end + end + + describe "remove_page_rule/2" do + test "is idempontent", %{site: site} do + {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"}) + assert remove_page_rule(site, rule.id) == :ok + refute Repo.get(Plausible.Shield.PageRule, rule.id) + assert remove_page_rule(site, rule.id) == :ok + end + end + + describe "list_page_rules/1" do + test "empty", %{site: site} do + assert(list_page_rules(site) == []) + end + + @tag :slow + test "many", %{site: site} do + {:ok, r1} = add_page_rule(site, %{"page_path" => "/test"}) + :timer.sleep(1000) + {:ok, r2} = add_page_rule(site, %{"page_path" => "/test"}) + assert [^r2, ^r1] = list_page_rules(site) + end + end + + describe "count_page_rules/1" do + test "counts", %{site: site} do + assert count_page_rules(site) == 0 + {:ok, _} = add_page_rule(site, %{"page_path" => "/test1"}) + assert count_page_rules(site) == 1 + {:ok, _} = add_page_rule(site, %{"page_path" => "/test2"}) + assert count_page_rules(site) == 2 + end + end + + describe "Page Rules" do + test "end to end", %{site: site} do + site2 = insert(:site) + + assert count_page_rules(site.id) == 0 + assert list_page_rules(site.id) == [] + + assert {:ok, rule} = + add_page_rule(site.id, %{ + "page_path" => "/test" + }) + + add_page_rule(site2, %{"page_path" => "/test"}) + + assert count_page_rules(site) == 1 + assert [%{id: rule_id}] = list_page_rules(site) + assert rule.id == rule_id + assert rule.page_path == "/test" + assert rule.action == :deny + refute rule.from_cache? + assert page_blocked?(site, "/test") + refute page_blocked?(site, "/testing") + end + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 436c391c7..77c9a9343 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -257,7 +257,7 @@ defmodule PlausibleWeb.SiteController do end def settings_shields(conn, %{"shield" => shield}) - when shield in ["ip_addresses", "countries"] do + when shield in ["ip_addresses", "countries", "pages"] do site = conn.assigns.site conn diff --git a/lib/plausible_web/live/shields/page_rules.ex b/lib/plausible_web/live/shields/page_rules.ex new file mode 100644 index 000000000..3ff38e12b --- /dev/null +++ b/lib/plausible_web/live/shields/page_rules.ex @@ -0,0 +1,304 @@ +defmodule PlausibleWeb.Live.Shields.PageRules do + @moduledoc """ + LiveView allowing page Rules management + """ + + use Phoenix.LiveComponent, global_prefixes: ~w(x-) + use Phoenix.HTML + + alias PlausibleWeb.Live.Components.Modal + alias Plausible.Shields + alias Plausible.Shield + + import PlausibleWeb.ErrorHelpers + + def update(assigns, socket) do + socket = + socket + |> assign( + page_rules_count: assigns[:page_rules_count] || socket.assigns.page_rules_count, + site: assigns[:site] || socket.assigns.site, + current_user: assigns[:current_user] || socket.assigns.current_user, + form: new_form() + ) + |> assign_new(:page_rules, fn %{site: site} -> + Shields.list_page_rules(site) + end) + |> assign_new(:redundant_rules, fn %{page_rules: page_rules} -> + detect_redundancy(page_rules) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+
+
+

+ Pages Block List +

+

+ Reject incoming traffic for specific pages +

+ + +
+
+
+ + + Add Page + +
+ = Shields.maximum_page_rules()} + class="mt-4" + title="Maximum number of pages reached" + > +

+ You've reached the maximum number of pages you can block (<%= Shields.maximum_page_rules() %>). Please remove one before adding another. +

+
+
+ + <.live_component module={Modal} id="page-rule-form-modal"> + <.form + :let={f} + for={@form} + phx-submit="save-page-rule" + phx-target={@myself} + class="max-w-md w-full mx-auto bg-white dark:bg-gray-800" + > +

Add Page to Block List

+ + <.live_component + submit_name="page_rule[page_path]" + submit_value={f[:page_path].value} + display_value={f[:page_path].value || ""} + module={PlausibleWeb.Live.Components.ComboBox} + suggest_fun={fn input, options -> suggest_page_paths(input, options, @site) end} + id={f[:page_path].id} + creatable + /> + + <%= error_tag(f, :page_path) %> + +

+ You can use a wildcard (*) to match multiple pages. For example, + /blog/* + will match /blog/post. + Once added, we will start rejecting traffic from this page within a few minutes. +

+
+ + Add Page → + +
+ + + +

+ No Page Rules configured for this Site. +

+
+ + + + + + + + + + <%= for rule <- @page_rules do %> + + + + + + + <% end %> + +
+ page + + Status + + Remove +
+
+ + <%= rule.page_path %> + +
+
+
+ + Blocked + + + Allowed + + + + + +
+
+ +
+
+
+
+ """ + end + + def handle_event("save-page-rule", %{"page_rule" => params}, socket) do + user = socket.assigns.current_user + + case Shields.add_page_rule( + socket.assigns.site.id, + params, + added_by: user + ) do + {:ok, rule} -> + page_rules = [rule | socket.assigns.page_rules] + + socket = + socket + |> Modal.close("page-rule-form-modal") + |> assign( + form: new_form(), + page_rules: page_rules, + page_rules_count: socket.assigns.page_rules_count + 1, + redundant_rules: detect_redundancy(page_rules) + ) + + # Make sure to clear the combobox input after adding a page rule, on subsequent modal reopening + send_update(PlausibleWeb.Live.Components.ComboBox, + id: "page_rule_page_code", + display_value: "" + ) + + send_flash( + :success, + "Page rule added successfully. Traffic will be rejected within a few minutes." + ) + + {:noreply, socket} + + {:error, %Ecto.Changeset{} = changeset} -> + {:noreply, assign(socket, form: to_form(changeset))} + end + end + + def handle_event("remove-page-rule", %{"rule-id" => rule_id}, socket) do + Shields.remove_page_rule(socket.assigns.site.id, rule_id) + + send_flash( + :success, + "Page rule removed successfully. Traffic will be resumed within a few minutes." + ) + + page_rules = Enum.reject(socket.assigns.page_rules, &(&1.id == rule_id)) + + {:noreply, + socket + |> assign( + page_rules_count: socket.assigns.page_rules_count - 1, + page_rules: page_rules, + redundant_rules: detect_redundancy(page_rules) + )} + end + + def send_flash(kind, message) do + send(self(), {:flash, kind, message}) + end + + defp new_form() do + %Shield.PageRule{} + |> Shield.PageRule.changeset(%{}) + |> to_form() + end + + defp format_added_at(dt, tz) do + dt + |> Plausible.Timezones.to_datetime_in_timezone(tz) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") + end + + def suggest_page_paths(input, _options, site) do + query = Plausible.Stats.Query.from(site, %{}) + + site + |> Plausible.Stats.filter_suggestions(query, "page", input) + |> Enum.map(fn %{label: label, value: value} -> {label, value} end) + end + + defp detect_redundancy(page_rules) do + page_rules + |> Enum.reduce(%{}, fn rule, acc -> + {[^rule], remaining_rules} = + Enum.split_with( + page_rules, + fn r -> r == rule end + ) + + conflicting = + remaining_rules + |> Enum.filter(fn candidate -> + rule + |> Map.fetch!(:page_path_pattern) + |> maybe_compile() + |> Regex.match?(candidate.page_path) + end) + |> Enum.map(& &1.id) + + Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc -> + Map.update(acc, conflicting_rule_id, [rule.page_path], fn existing -> + [rule.page_path | existing] + end) + end) + end) + end + + defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern) + defp maybe_compile(%Regex{} = pattern), do: pattern +end diff --git a/lib/plausible_web/live/shields/pages.ex b/lib/plausible_web/live/shields/pages.ex new file mode 100644 index 000000000..cd42c47b3 --- /dev/null +++ b/lib/plausible_web/live/shields/pages.ex @@ -0,0 +1,53 @@ +defmodule PlausibleWeb.Live.Shields.Pages do + @moduledoc """ + LiveView for IP Addresses Shield + """ + use PlausibleWeb, :live_view + use Phoenix.HTML + + alias Plausible.Shields + alias Plausible.Sites + + def mount( + _params, + %{ + "domain" => domain, + "current_user_id" => user_id + }, + socket + ) do + socket = + socket + |> assign_new(:site, fn -> + Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin]) + end) + |> assign_new(:page_rules_count, fn %{site: site} -> + Shields.count_page_rules(site) + end) + |> assign_new(:current_user, fn -> + Plausible.Repo.get(Plausible.Auth.User, user_id) + end) + + {:ok, socket} + end + + def render(assigns) do + ~H""" +
+ <.flash_messages flash={@flash} /> + <.live_component + module={PlausibleWeb.Live.Shields.PageRules} + current_user={@current_user} + page_rules_count={@page_rules_count} + site={@site} + id="page-rules-#{@current_user.id}" + /> +
+ """ + end + + def handle_info({:flash, kind, message}, socket) do + socket = put_live_flash(socket, kind, message) + {:noreply, socket} + end +end diff --git a/lib/plausible_web/templates/site/settings_shields.html.heex b/lib/plausible_web/templates/site/settings_shields.html.heex index 411c39fd3..489d49daf 100644 --- a/lib/plausible_web/templates/site/settings_shields.html.heex +++ b/lib/plausible_web/templates/site/settings_shields.html.heex @@ -14,4 +14,11 @@ "domain" => @site.domain } ) %> + <% "pages" -> %> + <%= live_render(@conn, PlausibleWeb.Live.Shields.Pages, + session: %{ + "site_id" => @site.id, + "domain" => @site.domain + } + ) %> <% end %> diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index 0b587379d..dad3cb9ae 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -65,10 +65,15 @@ defmodule PlausibleWeb.LayoutView do %{ key: "Shields", icon: :shield_exclamation, - value: [ - %{key: "IP Addresses", value: "shields/ip_addresses"}, - %{key: "Countries", value: "shields/countries"} - ] + value: + [ + %{key: "IP Addresses", value: "shields/ip_addresses"}, + %{key: "Countries", value: "shields/countries"}, + if FunWithFlags.enabled?(:shield_pages, for: conn.assigns[:site]) do + %{key: "Pages", value: "shields/pages"} + end + ] + |> Enum.reject(&is_nil/1) }, %{key: "Email Reports", value: "email-reports", icon: :envelope}, if conn.assigns[:current_user_role] == :owner do diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index e1b499f76..da7b842d9 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -11,6 +11,7 @@ # and so on) as they will fail if something goes wrong. FunWithFlags.enable(:imports_exports) +FunWithFlags.enable(:shield_pages) user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index b6627d8bc..8714e0848 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -158,6 +158,25 @@ defmodule Plausible.Ingestion.EventTest do assert dropped.drop_reason == :site_country_blocklist end + test "event pipeline drops a request when page is on blocklist" do + site = insert(:site) + + payload = %{ + name: "pageview", + url: "http://dummy.site/blocked/page", + domain: site.domain + } + + conn = build_conn(:post, "/api/events", payload) + + {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/blocked/**"}) + + assert {:ok, request} = Request.build(conn) + + assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request) + assert dropped.drop_reason == :site_page_blocklist + end + test "event pipeline drops events for site with accept_trafic_until in the past" do yesterday = Date.add(Date.utc_today(), -1) diff --git a/test/plausible/shield/page_rule_cache_test.exs b/test/plausible/shield/page_rule_cache_test.exs new file mode 100644 index 000000000..ba26f3cf9 --- /dev/null +++ b/test/plausible/shield/page_rule_cache_test.exs @@ -0,0 +1,107 @@ +defmodule Plausible.Shield.PageRuleCacheTest do + use Plausible.DataCase, async: true + + alias Plausible.Shield.PageRule + alias Plausible.Shield.PageRuleCache + alias Plausible.Shields + + describe "public cache interface" do + test "cache caches page rules", %{test: test} do + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = + Supervisor.start_link( + [ + {PageRuleCache, + [cache_name: test, child_id: :page_rules_cache_id, ets_options: [:bag]]} + ], + strategy: :one_for_one, + name: :"cache_supervisor_#{test}" + ) + + site1 = insert(:site) + site2 = insert(:site) + + {:ok, %{id: rid1}} = Shields.add_page_rule(site1, %{"page_path" => "/test/1"}) + {:ok, %{id: rid2}} = Shields.add_page_rule(site1, %{"page_path" => "/test/2"}) + {:ok, %{id: rid3}} = Shields.add_page_rule(site2, %{"page_path" => "/test/2"}) + + :ok = PageRuleCache.refresh_all(cache_name: test) + + :ok = Shields.remove_page_rule(site1, rid1) + + assert PageRuleCache.size(test) == 3 + + # the rule order should be deterministic, but with 1s timestamp sorting precision + # race conditions may happen during tests + assert rules = PageRuleCache.get(site1.domain, cache_opts) + rule_ids = Enum.map(rules, & &1.id) + assert rid1 in rule_ids + assert rid2 in rule_ids + assert length(rules) == 2 + + assert %PageRule{from_cache?: true, id: ^rid3} = + PageRuleCache.get(site2.domain, cache_opts) + + refute PageRuleCache.get("rogue.example.com", cache_opts) + end + + test "page path patterns are already compiled when fetched from cache", %{test: test} do + site = insert(:site) + + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + + {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/hello/**/world"}) + :ok = PageRuleCache.refresh_all(cache_name: test) + assert regex = PageRuleCache.get(site.domain, cache_opts).page_path_pattern + assert regex == ~r/^\/hello\/.*\/world$/ + end + + test "cache allows lookups for page paths on sites with changed domain", %{test: test} do + {:ok, _} = start_test_cache(test) + cache_opts = [force?: true, cache_name: test] + site = insert(:site, domain: "new.example.com", domain_changed_from: "old.example.com") + + {:ok, _} = Shields.add_page_rule(site, %{"page_path" => "/#{test}"}) + :ok = PageRuleCache.refresh_all(cache_name: test) + + assert PageRuleCache.get("old.example.com", cache_opts) + assert PageRuleCache.get("new.example.com", cache_opts) + refute PageRuleCache.get("rogue.example.com", cache_opts) + end + + test "refreshes only recently added pages rules", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain = "site1.example.com" + site = insert(:site, domain: domain) + + cache_opts = [cache_name: test, force?: true] + + yesterday = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-1 * 60 * 60 * 24) + |> NaiveDateTime.truncate(:second) + + {:ok, r1} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/1"}) + Ecto.Changeset.change(r1, inserted_at: yesterday, updated_at: yesterday) |> Repo.update!() + {:ok, _} = Plausible.Shields.add_page_rule(site, %{"page_path" => "/test/2"}) + + assert PageRuleCache.get(domain, cache_opts) == nil + + assert :ok = PageRuleCache.refresh_updated_recently(cache_opts) + + assert %{page_path_pattern: ~r[^/test/2$]} = PageRuleCache.get(domain, cache_opts) + + assert :ok = PageRuleCache.refresh_all(cache_opts) + + assert [_, _] = PageRuleCache.get(domain, cache_opts) + end + end + + defp start_test_cache(cache_name) do + %{start: {m, f, a}} = PageRuleCache.child_spec(cache_name: cache_name, ets_options: [:bag]) + apply(m, f, a) + end +end diff --git a/test/plausible/shields_test.exs b/test/plausible/shields_test.exs index c39da4587..e69de29bb 100644 --- a/test/plausible/shields_test.exs +++ b/test/plausible/shields_test.exs @@ -1,270 +0,0 @@ -defmodule Plausible.ShieldsTest do - use Plausible.DataCase - import Plausible.Shields - - setup do - site = insert(:site) - {:ok, %{site: site}} - end - - describe "add_ip_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{}) - assert changeset.errors == [inet: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported netmask", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"}) - assert changeset.errors == [inet: {"netmask unsupported", []}] - refute changeset.valid? - assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"}) - end - - test "incorrect ip", %{site: site} do - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"}) - - assert changeset.errors == [ - inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]} - ] - - refute changeset.valid? - end - - test "non-strict IPs", %{site: site} do - assert {:error, _} = add_ip_rule(site, %{"inet" => "111"}) - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"}) - refute changeset.valid? - - assert changeset.errors == [ - inet: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_ip_site_id_inet_index"} - ]} - ] - end - - test "ipv6", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "ipv4", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}) - - assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id) - end - - test "over limit", %{site: site} do - for i <- 1..maximum_ip_rules() do - assert {:ok, _} = - add_ip_rule(site, %{"inet" => "1.1.1.#{i}"}) - end - - assert count_ip_rules(site) == maximum_ip_rules() - - assert {:error, changeset} = - add_ip_rule(site, %{"inet" => "1.1.1.31"}) - - refute changeset.valid? - assert changeset.errors == [inet: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_ip_rule(site, %{"inet" => "1.1.1.1"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_ip_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert remove_ip_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.IPRule, rule.id) - assert remove_ip_rule(site, rule.id) == :ok - end - end - - describe "list_ip_rules/1" do - test "empty", %{site: site} do - assert(list_ip_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - :timer.sleep(1000) - {:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert [^r2, ^r1] = list_ip_rules(site) - end - end - - describe "count_ip_rules/1" do - test "counts", %{site: site} do - assert count_ip_rules(site) == 0 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"}) - assert count_ip_rules(site) == 1 - {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"}) - assert count_ip_rules(site) == 2 - end - end - - describe "IP Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_ip_rules(site.id) == 0 - assert list_ip_rules(site.id) == [] - - assert {:ok, rule} = - add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"}) - - assert count_ip_rules(site) == 1 - assert [^rule] = list_ip_rules(site) - assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32} - assert rule.description == "Localhost" - assert rule.action == :deny - refute rule.from_cache? - end - end - - describe "add_country_rule/2" do - test "no input", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{}) - assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}] - refute changeset.valid? - end - - test "unsupported country", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"}) - assert changeset.errors == [country_code: {"is invalid", []}] - refute changeset.valid? - end - - test "incorrect country format", %{site: site} do - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"}) - - assert changeset.errors == - [ - {:country_code, {"is invalid", []}}, - {:country_code, - {"should be %{count} character(s)", - [count: 2, validation: :length, kind: :is, type: :string]}} - ] - - refute changeset.valid? - end - - test "double insert", %{site: site} do - assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"}) - refute changeset.valid? - - assert changeset.errors == [ - country_code: - {"has already been taken", - [ - {:constraint, :unique}, - {:constraint_name, "shield_rules_country_site_id_country_code_index"} - ]} - ] - end - - test "over limit", %{site: site} do - country_codes = - Location.Country.all() - |> Enum.take(Plausible.Shields.maximum_country_rules()) - |> Enum.map(& &1.alpha_2) - - for cc <- country_codes do - assert {:ok, _} = - add_country_rule(site, %{"country_code" => cc}) - end - - assert count_country_rules(site) == maximum_country_rules() - - assert {:error, changeset} = - add_country_rule(site, %{"country_code" => "US"}) - - refute changeset.valid? - assert changeset.errors == [country_code: {"maximum reached", []}] - end - - test "with added_by", %{site: site} do - assert {:ok, rule} = - add_country_rule(site, %{"country_code" => "EE"}, - added_by: build(:user, name: "Joe", email: "joe@example.com") - ) - - assert rule.added_by == "Joe " - end - end - - describe "remove_country_rule/2" do - test "is idempontent", %{site: site} do - {:ok, rule} = add_country_rule(site, %{"country_code" => "EE"}) - assert remove_country_rule(site, rule.id) == :ok - refute Repo.get(Plausible.Shield.CountryRule, rule.id) - assert remove_country_rule(site, rule.id) == :ok - end - end - - describe "list_country_rules/1" do - test "empty", %{site: site} do - assert(list_country_rules(site) == []) - end - - @tag :slow - test "many", %{site: site} do - {:ok, r1} = add_country_rule(site, %{"country_code" => "EE"}) - :timer.sleep(1000) - {:ok, r2} = add_country_rule(site, %{"country_code" => "PL"}) - assert [^r2, ^r1] = list_country_rules(site) - end - end - - describe "count_country_rules/1" do - test "counts", %{site: site} do - assert count_country_rules(site) == 0 - {:ok, _} = add_country_rule(site, %{"country_code" => "EE"}) - assert count_country_rules(site) == 1 - {:ok, _} = add_country_rule(site, %{"country_code" => "PL"}) - assert count_country_rules(site) == 2 - end - end - - describe "Country Rules" do - test "end to end", %{site: site} do - site2 = insert(:site) - - assert count_country_rules(site.id) == 0 - assert list_country_rules(site.id) == [] - - assert {:ok, rule} = - add_country_rule(site.id, %{"country_code" => "EE"}) - - add_country_rule(site2, %{"country_code" => "PL"}) - - assert count_country_rules(site) == 1 - assert [^rule] = list_country_rules(site) - assert rule.country_code == "EE" - assert rule.action == :deny - refute rule.from_cache? - end - end -end diff --git a/test/plausible_web/live/shields/ip_addresses_test.exs b/test/plausible_web/live/shields/ip_addresses_test.exs index fd11a431d..16a4f2b1a 100644 --- a/test/plausible_web/live/shields/ip_addresses_test.exs +++ b/test/plausible_web/live/shields/ip_addresses_test.exs @@ -95,7 +95,7 @@ defmodule PlausibleWeb.Live.Shields.IPAddressesTest do assert element_exists?(html, ~s/a[phx-click="prefill-own-ip-rule"]/) end - test "form modal does not contains link to add own IP if already added", %{ + test "form modal does not contain link to add own IP if already added", %{ site: site, conn: conn } do diff --git a/test/plausible_web/live/shields/pages_test.exs b/test/plausible_web/live/shields/pages_test.exs new file mode 100644 index 000000000..7f89456a8 --- /dev/null +++ b/test/plausible_web/live/shields/pages_test.exs @@ -0,0 +1,176 @@ +defmodule PlausibleWeb.Live.Shields.PagesTest do + use PlausibleWeb.ConnCase, async: true + + import Phoenix.LiveViewTest + import Plausible.Test.Support.HTML + + alias Plausible.Shields + + setup [:create_user, :create_site, :log_in] + + describe "Page Rules - static" do + test "renders page rules page with empty list", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert resp =~ "No Page Rules configured for this Site" + assert resp =~ "Pages Block List" + end + + test "lists page rules with remove actions", %{conn: conn, site: site} do + {:ok, r1} = + Shields.add_page_rule(site, %{"page_path" => "/test/1"}) + + {:ok, r2} = + Shields.add_page_rule(site, %{"page_path" => "/test/2"}) + + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert resp =~ "/test/1" + assert resp =~ "/test/2" + + assert remove_button_1 = find(resp, "#remove-page-rule-#{r1.id}") + assert remove_button_2 = find(resp, "#remove-page-rule-#{r2.id}") + + assert text_of_attr(remove_button_1, "phx-click" == "remove-page-rule") + assert text_of_attr(remove_button_1, "phx-value-rule-id" == r1.id) + assert text_of_attr(remove_button_2, "phx-click" == "remove-page-rule") + assert text_of_attr(remove_button_2, "phx-value-rule-id" == r2.id) + end + + test "add rule button is rendered", %{conn: conn, site: site} do + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + assert element_exists?(resp, ~s/button#add-page-rule[x-data]/) + attr = text_of_attr(resp, ~s/button#add-page-rule/, "x-on:click") + + assert attr =~ "open-modal" + assert attr =~ "page-rule-form-modal" + end + + test "add rule button is not rendered when maximum reached", %{conn: conn, site: site} do + for i <- 1..Shields.maximum_page_rules() do + assert {:ok, _} = + Shields.add_page_rule(site, %{"page_path" => "/test/#{i}"}) + end + + conn = get(conn, "/#{site.domain}/settings/shields/pages") + resp = html_response(conn, 200) + + refute element_exists?(resp, ~s/button#add-page-rule[x-data]/) + assert resp =~ "Maximum number of pages reached" + assert resp =~ "You've reached the maximum number of pages you can block (30)" + end + end + + describe "Page Rules - LiveView" do + test "modal contains form", %{site: site, conn: conn} do + lv = get_liveview(conn, site) + html = render(lv) + + assert element_exists?( + html, + ~s/form[phx-submit="save-page-rule"] input[name="page_rule\[page_path\]"]/ + ) + + assert submit_button(html, ~s/form[phx-submit="save-page-rule"]/) + end + + test "submitting a valid Page saves it", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/**" + }) + + html = render(lv) + + assert html =~ "/test/**" + + assert [%{page_path: "/test/**"}] = Shields.list_page_rules(site) + end + + test "submitting invalid Page renders error", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "WRONG" + }) + + html = render(lv) + assert html =~ "must start with /" + end + + test "clicking Remove deletes the rule", %{conn: conn, site: site} do + {:ok, _} = + Shields.add_page_rule(site, %{"page_path" => "/test/*/page"}) + + lv = get_liveview(conn, site) + + html = render(lv) + assert html =~ "/test/*/page" + + lv |> element(~s/button[phx-click="remove-page-rule"]/) |> render_click() + + html = render(lv) + refute html =~ "/test/*/page" + + assert Shields.count_page_rules(site) == 0 + end + + test "conclicting rules are annotated with a warning", %{conn: conn, site: site} do + lv = get_liveview(conn, site) + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/*" + }) + + html = render(lv) + refute html =~ "This rule might be redundant" + + lv + |> element("form") + |> render_submit(%{ + "page_rule[page_path]" => "/test/another" + }) + + html = render(lv) + + assert html =~ "/test/*" + assert html =~ "/test/another" + + assert html =~ + "This rule might be redundant because the following rules may match first:\n\n/test/*" + + broader_rule_id = + site + |> Shields.list_page_rules() + |> Enum.find(&(&1.page_path == "/test/*")) + |> Map.fetch!(:id) + + lv |> element(~s/button#remove-page-rule-#{broader_rule_id}/) |> render_click() + html = render(lv) + + assert html =~ "/test/another" + refute html =~ "/test/*" + + refute html =~ + "This rule might be redundant because the following rules may match first:\n\n/test/*" + end + + defp get_liveview(conn, site) do + conn = assign(conn, :live_module, PlausibleWeb.Live.Shields) + {:ok, lv, _html} = live(conn, "/#{site.domain}/settings/shields/pages") + + lv + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 75e431245..78171693a 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -3,6 +3,7 @@ Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) Application.ensure_all_started(:double) FunWithFlags.enable(:window_time_on_page) FunWithFlags.enable(:imports_exports) +FunWithFlags.enable(:shield_pages) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) if Mix.env() == :small_test do From 4af7019011f5ef21928c98e9c8442786ccc61521 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 09:01:07 +0200 Subject: [PATCH 4/9] Ignore sessions without entry/exit pages when breaking down entry/exit pages (#3933) * Ignore sessions without entry/exit pages when breaking down entry/exit pages * Update stats controller tests to have more realistic test data (pageview followed by event) --- CHANGELOG.md | 1 + lib/plausible/stats/breakdown.ex | 4 +++ .../CSVs/30d-filter-goal/entry_pages.csv | 2 +- .../CSVs/30d-filter-goal/exit_pages.csv | 2 +- .../controllers/CSVs/30d/custom_props.csv | 2 +- .../controllers/CSVs/30d/entry_pages.csv | 2 +- .../controllers/CSVs/30d/exit_pages.csv | 2 +- .../controllers/CSVs/30d/pages.csv | 1 + .../controllers/CSVs/30d/referrers.csv | 2 +- .../controllers/CSVs/30d/sources.csv | 2 +- .../controllers/CSVs/30d/visitors.csv | 2 +- .../controllers/CSVs/6m/custom_props.csv | 2 +- .../controllers/CSVs/6m/entry_pages.csv | 2 +- .../controllers/CSVs/6m/exit_pages.csv | 2 +- .../controllers/CSVs/6m/pages.csv | 1 + .../controllers/CSVs/6m/referrers.csv | 2 +- .../controllers/CSVs/6m/sources.csv | 2 +- .../controllers/CSVs/6m/visitors.csv | 2 +- .../api/stats_controller/pages_test.exs | 36 +++++++++++++++++++ .../controllers/stats_controller_test.exs | 12 ++++++- 20 files changed, 68 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9263eb8d..a7742e03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ All notable changes to this project will be documented in this file. - Validate metric isn't queried multiple times ### Fixed +- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns - Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown - Fix `conversion_rate` metric in a `browser_versions` breakdown - Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 190d4cc4f..44d3fabf2 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -529,6 +529,8 @@ defmodule Plausible.Stats.Breakdown do defp do_group_by(q, "visit:entry_page") do from( s in q, + # Sessions without pageviews don't get entry_page assigned, hence they should get ignored + where: s.entry_page != "", group_by: s.entry_page, select_merge: %{entry_page: s.entry_page}, order_by: {:asc, s.entry_page} @@ -538,6 +540,8 @@ defmodule Plausible.Stats.Breakdown do defp do_group_by(q, "visit:exit_page") do from( s in q, + # Sessions without pageviews don't get entry_page assigned, hence they should get ignored + where: s.entry_page != "", group_by: s.exit_page, select_merge: %{exit_page: s.exit_page}, order_by: {:asc, s.exit_page} diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv index 2859d8c87..0f6f2cfa5 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/entry_pages.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,100.0 +/signup,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv index 2859d8c87..0f6f2cfa5 100644 --- a/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d-filter-goal/exit_pages.csv @@ -1,2 +1,2 @@ name,conversions,conversion_rate -,1,100.0 +/signup,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d/custom_props.csv b/test/plausible_web/controllers/CSVs/30d/custom_props.csv index 22ccc27a4..a0733de68 100644 --- a/test/plausible_web/controllers/CSVs/30d/custom_props.csv +++ b/test/plausible_web/controllers/CSVs/30d/custom_props.csv @@ -1,3 +1,3 @@ property,value,visitors,events,percentage variant,(none),3,4,75.0 -variant,A,1,1,25.0 +variant,A,1,2,25.0 diff --git a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv index a512fe4c3..59330a59c 100644 --- a/test/plausible_web/controllers/CSVs/30d/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/entry_pages.csv @@ -1,3 +1,3 @@ name,unique_entrances,total_entrances,visit_duration /,3,3,20 -,1,1,0 +/signup,1,1,60 diff --git a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv index a48eaed31..036337efe 100644 --- a/test/plausible_web/controllers/CSVs/30d/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/exit_pages.csv @@ -1,4 +1,4 @@ name,unique_exits,total_exits,exit_rate /,2,2,66.0 -,1,1, +/signup,1,1,100.0 /some-other-page,1,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv index 773bca0e3..e2990caa9 100644 --- a/test/plausible_web/controllers/CSVs/30d/pages.csv +++ b/test/plausible_web/controllers/CSVs/30d/pages.csv @@ -1,3 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,4,3,67, +/signup,1,1,0,60.0 /some-other-page,1,1,,60.0 diff --git a/test/plausible_web/controllers/CSVs/30d/referrers.csv b/test/plausible_web/controllers/CSVs/30d/referrers.csv index 12abdf9cf..b8a074551 100644 --- a/test/plausible_web/controllers/CSVs/30d/referrers.csv +++ b/test/plausible_web/controllers/CSVs/30d/referrers.csv @@ -1,2 +1,2 @@ name,visitors,bounce_rate,visit_duration -Direct / None,4,75,15 +Direct / None,4,50,30 diff --git a/test/plausible_web/controllers/CSVs/30d/sources.csv b/test/plausible_web/controllers/CSVs/30d/sources.csv index ac9093db0..f9d0b8c92 100644 --- a/test/plausible_web/controllers/CSVs/30d/sources.csv +++ b/test/plausible_web/controllers/CSVs/30d/sources.csv @@ -1,3 +1,3 @@ name,visitors,bounce_rate,visit_duration -Direct / None,3,100,0 +Direct / None,3,67,20 Google,1,0,60 diff --git a/test/plausible_web/controllers/CSVs/30d/visitors.csv b/test/plausible_web/controllers/CSVs/30d/visitors.csv index fcfe6aa24..6bac77976 100644 --- a/test/plausible_web/controllers/CSVs/30d/visitors.csv +++ b/test/plausible_web/controllers/CSVs/30d/visitors.csv @@ -28,5 +28,5 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration 2021-10-16,0,0,0,0.0,, 2021-10-17,0,0,0,0.0,, 2021-10-18,0,0,0,0.0,, -2021-10-19,2,1,2,0.5,100,0 +2021-10-19,2,2,2,1.0,50,30 2021-10-20,1,2,1,2.0,0,60 diff --git a/test/plausible_web/controllers/CSVs/6m/custom_props.csv b/test/plausible_web/controllers/CSVs/6m/custom_props.csv index 46a4e35d1..8c6ddd707 100644 --- a/test/plausible_web/controllers/CSVs/6m/custom_props.csv +++ b/test/plausible_web/controllers/CSVs/6m/custom_props.csv @@ -1,3 +1,3 @@ property,value,visitors,events,percentage variant,(none),4,5,80.0 -variant,A,1,1,20.0 +variant,A,1,2,20.0 diff --git a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv index 5ad9f3bf1..d0f98e949 100644 --- a/test/plausible_web/controllers/CSVs/6m/entry_pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/entry_pages.csv @@ -1,3 +1,3 @@ name,unique_entrances,total_entrances,visit_duration /,4,4,15 -,1,1,0 +/signup,1,1,60 diff --git a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv index 44805add9..bb235645a 100644 --- a/test/plausible_web/controllers/CSVs/6m/exit_pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/exit_pages.csv @@ -1,4 +1,4 @@ name,unique_exits,total_exits,exit_rate /,3,3,75.0 -,1,1, +/signup,1,1,100.0 /some-other-page,1,1,100.0 diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv index a61e49779..805158544 100644 --- a/test/plausible_web/controllers/CSVs/6m/pages.csv +++ b/test/plausible_web/controllers/CSVs/6m/pages.csv @@ -1,3 +1,4 @@ name,visitors,pageviews,bounce_rate,time_on_page /,5,4,75, +/signup,1,1,0,60.0 /some-other-page,1,1,,60.0 diff --git a/test/plausible_web/controllers/CSVs/6m/referrers.csv b/test/plausible_web/controllers/CSVs/6m/referrers.csv index 85b5ad5e7..fc901f771 100644 --- a/test/plausible_web/controllers/CSVs/6m/referrers.csv +++ b/test/plausible_web/controllers/CSVs/6m/referrers.csv @@ -1,2 +1,2 @@ name,visitors,bounce_rate,visit_duration -Direct / None,5,80,12 +Direct / None,5,60,24 diff --git a/test/plausible_web/controllers/CSVs/6m/sources.csv b/test/plausible_web/controllers/CSVs/6m/sources.csv index a1d2a9fa4..259c3a23f 100644 --- a/test/plausible_web/controllers/CSVs/6m/sources.csv +++ b/test/plausible_web/controllers/CSVs/6m/sources.csv @@ -1,3 +1,3 @@ name,visitors,bounce_rate,visit_duration -Direct / None,3,100,0 +Direct / None,3,67,20 Google,2,50,30 diff --git a/test/plausible_web/controllers/CSVs/6m/visitors.csv b/test/plausible_web/controllers/CSVs/6m/visitors.csv index b6881dd9e..230cc9d82 100644 --- a/test/plausible_web/controllers/CSVs/6m/visitors.csv +++ b/test/plausible_web/controllers/CSVs/6m/visitors.csv @@ -4,4 +4,4 @@ date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration 2021-07-01,0,0,0,0.0,, 2021-08-01,0,0,0,0.0,, 2021-09-01,1,1,1,1.0,100,0 -2021-10-01,3,3,3,1.0,67,20 +2021-10-01,3,4,3,1.33,33,40 diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 976745ee5..5a7a0bd38 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -1324,6 +1324,24 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do } ] end + + test "ignores entry pages from sessions with only custom events", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:15:00], + pathname: "/" + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [] + end end describe "GET /api/stats/:domain/exit-pages" do @@ -1557,5 +1575,23 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do %{"name" => "/exit2", "visitors" => 1, "visits" => 1} ] end + + test "ignores exit pages from sessions with only custom events", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:15:00], + pathname: "/" + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [] + end end end diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 39d59c4cb..fd48e9cd9 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -285,7 +285,7 @@ defmodule PlausibleWeb.StatsControllerTest do ["2021-09-27", "0", "0", "0", "0.0", "", ""], ["2021-10-04", "0", "0", "0", "0.0", "", ""], ["2021-10-11", "0", "0", "0", "0.0", "", ""], - ["2021-10-18", "3", "3", "3", "1.0", "67", "20"], + ["2021-10-18", "3", "4", "3", "1.33", "33", "40"], [""] ] end @@ -469,7 +469,17 @@ defmodule PlausibleWeb.StatsControllerTest do browser: "FirefoxNoVersion", operating_system: "MacNoVersion" ), + build(:pageview, + user_id: 456, + timestamp: + Timex.shift(~N[2021-10-20 12:00:00], days: -1, minutes: -1) + |> NaiveDateTime.truncate(:second), + pathname: "/signup", + "meta.key": ["variant"], + "meta.value": ["A"] + ), build(:event, + user_id: 456, timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1) |> NaiveDateTime.truncate(:second), name: "Signup", From 604bf884513ab45f9ee44d2a55025bc2f874b430 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 26 Mar 2024 10:29:55 +0200 Subject: [PATCH 5/9] Add github action to validate whether migrations and app change at the same time (#3945) --- .github/workflows/migrations-validation.yml | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/migrations-validation.yml diff --git a/.github/workflows/migrations-validation.yml b/.github/workflows/migrations-validation.yml new file mode 100644 index 000000000..2d20350b8 --- /dev/null +++ b/.github/workflows/migrations-validation.yml @@ -0,0 +1,28 @@ +name: Validate migrations + +on: + pull_request: + paths: + - 'priv/repo/migrations/**' + - 'priv/ingest_repo/migrations/**' + +jobs: + validate: + name: App code does not change at the same time + runs-on: ubuntu-latest + + steps: + - uses: dorny/paths-filter@v3 + id: changes + with: + list-files: json + filters: | + lib: + - 'lib/**' + - 'extra/**' + - 'config/**' + + - if: steps.changes.outputs.lib == 'true' + run: | + echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time" + exit 1 From 7523abe93ea62d3cb0b569abd67e8b5c18ec9811 Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 26 Mar 2024 09:42:48 +0100 Subject: [PATCH 6/9] Add metrics to ingestion pipeline (#3927) * Add metrics to ingestion pipeline * Format * Format * Update buckets * Credo --- lib/plausible/cache.ex | 24 +++------ lib/plausible/ingestion/event.ex | 54 ++++++++++++-------- lib/plausible/telemetry/plausible_metrics.ex | 23 +++++++++ 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.ex index 17ec2d2cf..c531b90c9 100644 --- a/lib/plausible/cache.ex +++ b/lib/plausible/cache.ex @@ -176,26 +176,16 @@ defmodule Plausible.Cache do defp refresh(mode, query, opts) when mode in @modes do cache_name = Keyword.get(opts, :cache_name, name()) - measure_duration(telemetry_event_refresh(cache_name, mode), fn -> - items = Plausible.Repo.all(query) - :ok = merge_items(items, opts) - end) + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_event_refresh(cache_name, mode), + fn -> + items = Plausible.Repo.all(query) + :ok = merge_items(items, opts) + end + ) :ok end - - defp measure_duration(event, fun) when is_function(fun, 0) do - {duration, result} = time_it(fun) - :telemetry.execute(event, %{duration: duration}, %{}) - result - end - - defp time_it(fun) do - start = System.monotonic_time() - result = fun.() - stop = System.monotonic_time() - {stop - start, result} - end end end end diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 6dd680f61..74ed95536 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -100,6 +100,10 @@ defmodule Plausible.Ingestion.Event do [:plausible, :ingest, :event, :dropped] end + def telemetry_pipeline_step_duration() do + [:plausible, :ingest, :pipeline, :step] + end + @spec emit_telemetry_buffered(t()) :: :ok def emit_telemetry_buffered(event) do :telemetry.execute(telemetry_event_buffered(), %{}, %{ @@ -119,34 +123,42 @@ defmodule Plausible.Ingestion.Event do defp pipeline() do [ - &drop_datacenter_ip/1, - &drop_shield_rule_ip/1, - &drop_shield_rule_page/1, - &put_geolocation/1, - &drop_shield_rule_country/1, - &put_user_agent/1, - &put_basic_info/1, - &put_referrer/1, - &put_utm_tags/1, - &put_props/1, - &put_revenue/1, - &put_salts/1, - &put_user_id/1, - &validate_clickhouse_event/1, - ®ister_session/1, - &write_to_buffer/1 + drop_datacenter_ip: &drop_datacenter_ip/1, + drop_shield_rule_page: &drop_shield_rule_page/1, + drop_shield_rule_ip: &drop_shield_rule_ip/1, + put_geolocation: &put_geolocation/1, + drop_shield_rule_country: &drop_shield_rule_country/1, + put_user_agent: &put_user_agent/1, + put_basic_info: &put_basic_info/1, + put_referrer: &put_referrer/1, + put_utm_tags: &put_utm_tags/1, + put_props: &put_props/1, + put_revenue: &put_revenue/1, + put_salts: &put_salts/1, + put_user_id: &put_user_id/1, + validate_clickhouse_event: &validate_clickhouse_event/1, + register_session: ®ister_session/1, + write_to_buffer: &write_to_buffer/1 ] end defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do - Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event -> - case pipeline_step.(acc_event) do - %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} - %__MODULE__{dropped?: false} = event -> {:cont, event} - end + Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event -> + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_pipeline_step_duration(), + fn -> execute_step(step_fn, acc_event) end, + %{step: "#{step_name}"} + ) end) end + defp execute_step(step_fn, acc_event) do + case step_fn.(acc_event) do + %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} + %__MODULE__{dropped?: false} = event -> {:cont, event} + end + end + defp new(domain, request) do struct!(__MODULE__, domain: domain, request: request) end diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex index 80e78d25b..17e24e337 100644 --- a/lib/plausible/telemetry/plausible_metrics.ex +++ b/lib/plausible/telemetry/plausible_metrics.ex @@ -47,6 +47,16 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do unit: {:native, :millisecond}, measurement: :duration ), + distribution( + metric_prefix ++ [:ingest, :events, :pipeline, :steps], + event_name: Ingestion.Event.telemetry_pipeline_step_duration(), + reporter_options: [ + buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000] + ], + unit: {:native, :microsecond}, + measurement: :duration, + tags: [:step] + ), counter( metric_prefix ++ [:ingest, :events, :buffered, :total], event_name: Ingestion.Event.telemetry_event_buffered() @@ -110,6 +120,19 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do }) end + def measure_duration(event, fun, meta \\ %{}) when is_function(fun, 0) do + {duration, result} = time_it(fun) + :telemetry.execute(event, %{duration: duration}, meta) + result + end + + defp time_it(fun) do + start = System.monotonic_time() + result = fun.() + stop = System.monotonic_time() + {stop - start, result} + end + defp write_buffer_metrics(metric_prefix, poll_rate) do Polling.build( :write_buffer_metrics, From edf70d14b6da8f7eeaebc4012550e06edb749be9 Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 26 Mar 2024 09:49:15 +0100 Subject: [PATCH 7/9] Use sessionStorage for "dashboard first launch" banner tracking (#3892) * Use sessionStorage for offer e-mail report banner tracking Keeping it within the cookie is problematic, as the banners don't expire and overflow the cookie with data when enough new sites are added. Ref https://github.com/plausible/analytics/issues/3762 * Update changelog * Extract a component * Make is_dbip evaluate to quoted boolean --- CHANGELOG.md | 1 + .../first_dashboard_launch_banner.ex | 50 +++++++++++++ .../controllers/site_controller.ex | 6 +- .../controllers/stats_controller.ex | 16 +--- .../templates/site/snippet.html.heex | 5 ++ .../templates/stats/stats.html.eex | 63 ---------------- .../templates/stats/stats.html.heex | 73 +++++++++++++++++++ .../controllers/site_controller_test.exs | 10 ++- 8 files changed, 140 insertions(+), 84 deletions(-) create mode 100644 lib/plausible_web/components/first_dashboard_launch_banner.ex delete mode 100644 lib/plausible_web/templates/stats/stats.html.eex create mode 100644 lib/plausible_web/templates/stats/stats.html.heex diff --git a/CHANGELOG.md b/CHANGELOG.md index a7742e03d..ba2a7cd1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ All notable changes to this project will be documented in this file. - Validate metric isn't queried multiple times ### Fixed +- Creating many sites no longer leads to cookie overflow - Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns - Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown - Fix `conversion_rate` metric in a `browser_versions` breakdown diff --git a/lib/plausible_web/components/first_dashboard_launch_banner.ex b/lib/plausible_web/components/first_dashboard_launch_banner.ex new file mode 100644 index 000000000..f4ed01c43 --- /dev/null +++ b/lib/plausible_web/components/first_dashboard_launch_banner.ex @@ -0,0 +1,50 @@ +defmodule PlausibleWeb.Components.FirstDashboardLaunchBanner do + @moduledoc """ + A banner that appears on the first dashboard launch + """ + use Phoenix.Component + use Phoenix.HTML + + attr(:site, Plausible.Site, required: true) + + def set(assigns) do + ~H""" + + """ + end + + attr(:site, Plausible.Site, required: true) + + def render(assigns) do + ~H""" + + """ + end + + defp x_data(site) do + "{show: !!sessionStorage.getItem('#{storage_key(site)}')}" + end + + defp x_init(site) do + "setTimeout(() => sessionStorage.removeItem('#{storage_key(site)}'), 3000)" + end + + defp storage_key(site) do + "dashboard_seen_#{site.domain}" + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 77c9a9343..64cc4332d 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -34,9 +34,9 @@ defmodule PlausibleWeb.SiteController do |> Plausible.Mailer.send() end - conn - |> put_session(site.domain <> "_offer_email_report", true) - |> redirect(external: Routes.site_path(conn, :add_snippet, site.domain)) + redirect(conn, + external: Routes.site_path(conn, :add_snippet, site.domain, site_created: true) + ) {:error, {:over_limit, limit}} -> render(conn, "new.html", diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 8e331f8a7..1e8d7f321 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -59,10 +59,7 @@ defmodule PlausibleWeb.StatsController do cond do stats_start_date && can_see_stats? -> - offer_email_report = get_session(conn, site.domain <> "_offer_email_report") - conn - |> remove_email_report_banner(site) |> put_resp_header("x-robots-tag", "noindex, nofollow") |> render("stats.html", site: site, @@ -72,7 +69,6 @@ defmodule PlausibleWeb.StatsController do stats_start_date: stats_start_date, native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at), title: title(conn, site), - offer_email_report: offer_email_report, demo: demo, flags: get_flags(conn.assigns[:current_user]), is_dbip: is_dbip(), @@ -328,7 +324,6 @@ defmodule PlausibleWeb.StatsController do stats_start_date: stats_start_date, native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at), title: title(conn, shared_link.site), - offer_email_report: false, demo: false, dogfood_page_path: "/share/:dashboard", shared_link_auth: shared_link.slug, @@ -351,14 +346,6 @@ defmodule PlausibleWeb.StatsController do end end - defp remove_email_report_banner(conn, site) do - if conn.assigns[:current_user] do - delete_session(conn, site.domain <> "_offer_email_report") - else - conn - end - end - defp shared_link_cookie_name(slug), do: "shared-link-" <> slug defp get_flags(_user) do @@ -367,11 +354,12 @@ defmodule PlausibleWeb.StatsController do defp is_dbip() do on_full_build do - false + "false" else Plausible.Geo.database_type() |> to_string() |> String.starts_with?("DBIP") + |> to_string() end end diff --git a/lib/plausible_web/templates/site/snippet.html.heex b/lib/plausible_web/templates/site/snippet.html.heex index 0035dfdc0..de462c0bf 100644 --- a/lib/plausible_web/templates/site/snippet.html.heex +++ b/lib/plausible_web/templates/site/snippet.html.heex @@ -1,3 +1,8 @@ + +
<%= form_for @conn, "/", [class: "max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>

Add JavaScript snippet

diff --git a/lib/plausible_web/templates/stats/stats.html.eex b/lib/plausible_web/templates/stats/stats.html.eex deleted file mode 100644 index 382ca9162..000000000 --- a/lib/plausible_web/templates/stats/stats.html.eex +++ /dev/null @@ -1,63 +0,0 @@ -
- <%= if @offer_email_report do %> - - <% end %> - - <%= if @site.locked do %> - - <% end %> - -
-
-
- - <%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %> -
-
-

- Want these stats for your website? -
- Start your free trial today. -

- -
-
- <% end %> -
diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heex new file mode 100644 index 000000000..a4b55810c --- /dev/null +++ b/lib/plausible_web/templates/stats/stats.html.heex @@ -0,0 +1,73 @@ +
+ + + <%= if @site.locked do %> + + <% end %> + +
+
Jason.encode!() + } + > +
+ + <%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %> +
+
+

+ Want these stats for your website?
+ Start your free trial today. +

+ +
+
+ <% end %> +
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index 12fe79abf..6b6d0cb22 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -244,7 +244,9 @@ defmodule PlausibleWeb.SiteControllerTest do } }) - assert redirected_to(conn) == "/#{URI.encode_www_form("éxample.com")}/snippet" + assert redirected_to(conn) == + "/#{URI.encode_www_form("éxample.com")}/snippet?site_created=true" + assert site = Repo.get_by(Plausible.Site, domain: "éxample.com") assert site.timezone == "Europe/London" assert site.ingest_rate_limit_scale_seconds == 60 @@ -341,7 +343,7 @@ defmodule PlausibleWeb.SiteControllerTest do } }) - assert redirected_to(conn) == "/example.com/snippet" + assert redirected_to(conn) == "/example.com/snippet?site_created=true" assert Repo.get_by(Plausible.Site, domain: "example.com") end @@ -361,7 +363,7 @@ defmodule PlausibleWeb.SiteControllerTest do } }) - assert redirected_to(conn) == "/example.com/snippet" + assert redirected_to(conn) == "/example.com/snippet?site_created=true" assert Plausible.Billing.Quota.site_usage(user) == 3 end @@ -375,7 +377,7 @@ defmodule PlausibleWeb.SiteControllerTest do } }) - assert redirected_to(conn) == "/example.com/snippet" + assert redirected_to(conn) == "/example.com/snippet?site_created=true" assert Repo.get_by(Plausible.Site, domain: "example.com") end end From b31433a7bf0cf55819dcfc4618de1889ca636399 Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 26 Mar 2024 11:01:59 +0100 Subject: [PATCH 8/9] Ensure all the react container attributes are strings (#3948) --- .../controllers/stats_controller.ex | 3 +- .../templates/stats/stats.html.heex | 22 +++++----- .../controllers/stats_controller_test.exs | 44 +++++++++++++++++-- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 1e8d7f321..46a92e7af 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -354,12 +354,11 @@ defmodule PlausibleWeb.StatsController do defp is_dbip() do on_full_build do - "false" + false else Plausible.Geo.database_type() |> to_string() |> String.starts_with?("DBIP") - |> to_string() end end diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heex index a4b55810c..2a09599a9 100644 --- a/lib/plausible_web/templates/stats/stats.html.heex +++ b/lib/plausible_web/templates/stats/stats.html.heex @@ -17,23 +17,25 @@ style="overflow-anchor: none;" data-domain={@site.domain} data-offset={Plausible.Site.tz_offset(@site)} - data-has-goals={@has_goals} - data-conversions-opted-out={Plausible.Billing.Feature.Goals.opted_out?(@site)} - data-funnels-opted-out={Plausible.Billing.Feature.Funnels.opted_out?(@site)} - data-props-opted-out={Plausible.Billing.Feature.Props.opted_out?(@site)} + data-has-goals={to_string(@has_goals)} + data-conversions-opted-out={to_string(Plausible.Billing.Feature.Goals.opted_out?(@site))} + data-funnels-opted-out={to_string(Plausible.Billing.Feature.Funnels.opted_out?(@site))} + data-props-opted-out={to_string(Plausible.Billing.Feature.Props.opted_out?(@site))} data-funnels-available={ - Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok + to_string(Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok) + } + data-props-available={ + to_string(Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok) } - data-props-available={Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok} data-funnels={Jason.encode!(@funnels)} - data-has-props={@has_props} - data-logged-in={!!@conn.assigns[:current_user]} + data-has-props={to_string(@has_props)} + data-logged-in={to_string(!!@conn.assigns[:current_user])} data-stats-begin={@stats_start_date} data-native-stats-begin={@native_stats_start_date} data-shared-link-auth={assigns[:shared_link_auth]} - data-embedded={@conn.assigns[:embedded]} + data-embedded={to_string(@conn.assigns[:embedded])} data-background={@conn.assigns[:background]} - data-is-dbip={@is_dbip} + data-is-dbip={to_string(@is_dbip)} data-current-user-role={@conn.assigns[:current_user_role]} data-flags={Jason.encode!(@flags)} data-valid-intervals-by-period={ diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index fd48e9cd9..80030ce4c 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -16,6 +16,18 @@ defmodule PlausibleWeb.StatsControllerTest do assert text_of_attr(resp, @react_container, "data-domain") == site.domain assert text_of_attr(resp, @react_container, "data-is-dbip") == "false" + assert text_of_attr(resp, @react_container, "data-has-goals") == "false" + assert text_of_attr(resp, @react_container, "data-conversions-opted-out") == "false" + assert text_of_attr(resp, @react_container, "data-funnels-opted-out") == "false" + assert text_of_attr(resp, @react_container, "data-props-opted-out") == "false" + assert text_of_attr(resp, @react_container, "data-props-available") == "true" + assert text_of_attr(resp, @react_container, "data-funnels-available") == "true" + assert text_of_attr(resp, @react_container, "data-has-props") == "false" + assert text_of_attr(resp, @react_container, "data-logged-in") == "false" + assert text_of_attr(resp, @react_container, "data-embedded") == "" + + [{"div", attrs, _}] = find(resp, @react_container) + assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end) assert ["noindex, nofollow"] == resp @@ -61,7 +73,8 @@ defmodule PlausibleWeb.StatsControllerTest do test "can view stats of a website I've created", %{conn: conn, site: site} do populate_stats(site, [build(:pageview)]) conn = get(conn, "/" <> site.domain) - assert html_response(conn, 200) =~ "stats-react-container" + resp = html_response(conn, 200) + assert text_of_attr(resp, @react_container, "data-logged-in") == "true" end test "shows locked page if page is locked", %{conn: conn, user: user} do @@ -102,8 +115,11 @@ defmodule PlausibleWeb.StatsControllerTest do populate_stats(site, [build(:pageview)]) conn = get(conn, "/" <> site.domain) - assert html_response(conn, 200) =~ "stats-react-container" - assert html_response(conn, 200) =~ "This dashboard is actually locked" + resp = html_response(conn, 200) + assert resp =~ "This dashboard is actually locked" + + [{"div", attrs, _}] = find(resp, @react_container) + assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end) end test "can view a private locked dashboard without stats", %{conn: conn} do @@ -120,7 +136,10 @@ defmodule PlausibleWeb.StatsControllerTest do populate_stats(site, [build(:pageview)]) conn = get(conn, "/" <> site.domain) - assert html_response(conn, 200) =~ "stats-react-container" + resp = html_response(conn, 200) + + [{"div", attrs, _}] = find(resp, @react_container) + assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end) end end @@ -616,9 +635,26 @@ defmodule PlausibleWeb.StatsControllerTest do link = insert(:shared_link, site: site) conn = get(conn, "/share/test-site.com/?auth=#{link.slug}") + resp = html_response(conn, 200) + assert text_of_attr(resp, @react_container, "data-embedded") == "false" assert Plug.Conn.get_resp_header(conn, "x-frame-options") == [] end + test "returns page embedded page", %{ + conn: conn + } do + site = insert(:site, domain: "test-site.com") + link = insert(:shared_link, site: site) + + conn = get(conn, "/share/test-site.com/?auth=#{link.slug}&embed=true") + resp = html_response(conn, 200) + assert text_of_attr(resp, @react_container, "data-embedded") == "true" + assert Plug.Conn.get_resp_header(conn, "x-frame-options") == [] + + [{"div", attrs, _}] = find(resp, @react_container) + assert Enum.all?(attrs, fn {k, v} -> is_binary(k) and is_binary(v) end) + end + test "shows locked page if page is locked", %{conn: conn} do site = insert(:site, domain: "test-site.com", locked: true) link = insert(:shared_link, site: site) From c263df5805339405e491787950114e59c3c25c05 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 26 Mar 2024 19:55:14 +0800 Subject: [PATCH 9/9] CSV imports (UI) (#3845) * add basic ui * remove TODO * credo * allow folder upload * redirect external * mention folder, use folder icon for file picker * back to multiple file upload * mention zip * escape dots in archive filename --- assets/js/liveview/live_socket.js | 20 +- lib/plausible/exports.ex | 16 +- lib/plausible/imported/csv_importer.ex | 49 +++- .../controllers/site_controller.ex | 11 +- lib/plausible_web/live/csv_import.ex | 228 ++++++++++++++++++ lib/plausible_web/router.ex | 3 +- .../templates/site/csv_import.html.heex | 20 ++ .../site/settings_imports_exports.html.heex | 4 +- 8 files changed, 340 insertions(+), 11 deletions(-) create mode 100644 lib/plausible_web/live/csv_import.ex create mode 100644 lib/plausible_web/templates/site/csv_import.html.heex diff --git a/assets/js/liveview/live_socket.js b/assets/js/liveview/live_socket.js index a6208ff4c..ed287267f 100644 --- a/assets/js/liveview/live_socket.js +++ b/assets/js/liveview/live_socket.js @@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) { }) } } + let Uploaders = {} + Uploaders.S3 = function (entries, onViewError) { + entries.forEach(entry => { + let xhr = new XMLHttpRequest() + onViewError(() => xhr.abort()) + xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error() + xhr.onerror = () => entry.error() + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + let percent = Math.round((event.loaded / event.total) * 100) + if (percent < 100) { entry.progress(percent) } + } + }) + let url = entry.meta.url + xhr.open("PUT", url, true) + xhr.send(entry.file) + }) + } let token = csrfToken.getAttribute("content") let url = websocketUrl.getAttribute("content") let liveUrl = (url === "") ? "/live" : new URL("/live", url).href; let liveSocket = new LiveSocket(liveUrl, Socket, { heartbeatIntervalMs: 10000, - params: { _csrf_token: token }, hooks: Hooks, dom: { + params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: { // for alpinejs integration onBeforeElUpdated(from, to) { if (from._x_dataStack) { diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex index dbabcd7ff..c15e0bb78 100644 --- a/lib/plausible/exports.ex +++ b/lib/plausible/exports.ex @@ -12,14 +12,24 @@ defmodule Plausible.Exports do Examples: iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31]) - "plausible.io_20210101_20241231.zip" + "plausible_io_20210101_20241231.zip" iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31]) - "Bücher.example_20210101_20241231.zip" + "Bücher_example_20210101_20241231.zip" """ def archive_filename(domain, min_date, max_date) do - "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip" + name = + Enum.join( + [ + String.replace(domain, ".", "_"), + Calendar.strftime(min_date, "%Y%m%d"), + Calendar.strftime(max_date, "%Y%m%d") + ], + "_" + ) + + name <> ".zip" end @doc """ diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 2d10ca32e..6376d851d 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -105,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do Date.range(~D[2019-01-01], ~D[2022-01-01]) iex> date_range([]) - ** (ArgumentError) empty uploads + nil """ - @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() + @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil) - def date_range([]), do: raise(ArgumentError, "empty uploads") + def date_range([]), do: nil defp date_range([upload | uploads], prev_start_date, prev_end_date) do filename = @@ -174,4 +174,47 @@ defmodule Plausible.Imported.CSVImporter do def parse_filename!(_filename) do raise ArgumentError, "invalid filename" end + + @doc """ + Checks if the provided filename conforms to the expected format. + + Examples: + + iex> valid_filename?("my_data.csv") + false + + iex> valid_filename?("imported_devices_00010101_20250101.csv") + true + + """ + @spec valid_filename?(String.t()) :: boolean + def valid_filename?(filename) do + try do + parse_filename!(filename) + else + _ -> true + rescue + _ -> false + end + end + + @doc """ + Extracts the table name from the provided filename. + + Raises if the filename doesn't conform to the expected format. + + Examples: + + iex> extract_table("my_data.csv") + ** (ArgumentError) invalid filename + + iex> extract_table("imported_devices_00010101_20250101.csv") + "imported_devices" + + """ + @spec extract_table(String.t()) :: String.t() + def extract_table(filename) do + {table, _start_date, _end_date} = parse_filename!(filename) + table + end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 64cc4332d..e61a26611 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -708,7 +708,7 @@ defmodule PlausibleWeb.SiteController do |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain)) end - def export(conn, _params) do + def csv_export(conn, _params) do %{site: site, current_user: user} = conn.assigns Oban.insert!( @@ -725,6 +725,15 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain)) end + def csv_import(conn, _params) do + conn + |> assign(:skip_plausible_tracking, true) + |> render("csv_import.html", + layout: {PlausibleWeb.LayoutView, "focus.html"}, + connect_live_socket: true + ) + end + def change_domain(conn, _params) do changeset = Plausible.Site.update_changeset(conn.assigns.site) diff --git a/lib/plausible_web/live/csv_import.ex b/lib/plausible_web/live/csv_import.ex new file mode 100644 index 000000000..38c97ca17 --- /dev/null +++ b/lib/plausible_web/live/csv_import.ex @@ -0,0 +1,228 @@ +defmodule PlausibleWeb.Live.CSVImport do + @moduledoc """ + LiveView allowing uploading CSVs for imported tables to S3 + """ + use PlausibleWeb, :live_view + alias Plausible.Imported.CSVImporter + + @impl true + def mount(_params, session, socket) do + %{"site_id" => site_id, "user_id" => user_id} = session + + socket = + socket + |> assign(site_id: site_id, user_id: user_id) + |> allow_upload(:import, + accept: [".csv", "text/csv"], + auto_upload: true, + max_entries: length(Plausible.Imported.tables()), + # 1GB + max_file_size: 1_000_000_000, + external: &presign_upload/2, + progress: &handle_progress/3 + ) + |> process_imported_tables() + + {:ok, socket} + end + + @impl true + def render(assigns) do + ~H""" +
+
+ <.csv_picker upload={@uploads.import} imported_tables={@imported_tables} /> + <.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} /> + +

+ <%= error_to_string(error) %> +

+
+
+ """ + end + + defp csv_picker(assigns) do + ~H""" + + """ + end + + defp confirm_button(assigns) do + ~H""" + + """ + end + + defp imported_table(assigns) do + status = + cond do + assigns.upload && assigns.upload.progress == 100 -> :success + assigns.upload && assigns.upload.progress > 0 -> :in_progress + not Enum.empty?(assigns.errors) -> :error + true -> :empty + end + + assigns = assign(assigns, status: status) + + ~H""" +
  • +
    + + + + + + + <%= if @upload do %> + <%= @upload.client_name %> + <% else %> + <%= @table %>_YYYYMMDD_YYYYMMDD.csv + <% end %> + +
    + +

    + <%= error_to_string(error) %> +

    +
  • + """ + end + + @impl true + def handle_event("validate-upload-form", _params, socket) do + {:noreply, process_imported_tables(socket)} + end + + def handle_event("submit-upload-form", _params, socket) do + %{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns + site = Plausible.Repo.get!(Plausible.Site, site_id) + user = Plausible.Repo.get!(Plausible.Auth.User, user_id) + + uploads = + consume_uploaded_entries(socket, :import, fn meta, entry -> + {:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}} + end) + + {:ok, _job} = + CSVImporter.new_import(site, user, + start_date: date_range.first, + end_date: date_range.last, + uploads: uploads + ) + + redirect_to = + Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain)) + + {:noreply, redirect(socket, external: redirect_to)} + end + + defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)" + defp error_to_string(:too_many_files), do: "too many files" + defp error_to_string(:not_accepted), do: "unacceptable file types" + defp error_to_string(:external_client_failure), do: "browser upload failed" + + defp presign_upload(entry, socket) do + %{s3_url: s3_url, presigned_url: upload_url} = + Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name) + + {:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket} + end + + defp handle_progress(:import, entry, socket) do + if entry.done? do + {:noreply, process_imported_tables(socket)} + else + {:noreply, socket} + end + end + + defp process_imported_tables(socket) do + tables = Plausible.Imported.tables() + {completed, in_progress} = uploaded_entries(socket, :import) + + {valid_uploads, invalid_uploads} = + Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name)) + + imported_tables_all_uploads = + Enum.map(tables, fn table -> + uploads = + Enum.filter(valid_uploads, fn upload -> + CSVImporter.extract_table(upload.client_name) == table + end) + + {upload, replaced_uploads} = List.pop_at(uploads, -1) + {table, upload, replaced_uploads} + end) + + imported_tables = + Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} -> + {table, upload} + end) + + replaced_uploads = + Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} -> + replaced_uploads + end) + + date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name)) + all_uploaded? = completed != [] and in_progress == [] + + socket + |> cancel_uploads(invalid_uploads) + |> cancel_uploads(replaced_uploads) + |> assign( + imported_tables: imported_tables, + can_confirm?: all_uploaded?, + date_range: date_range + ) + end + + defp cancel_uploads(socket, uploads) do + Enum.reduce(uploads, socket, fn upload, socket -> + cancel_upload(socket, :import, upload.ref) + end) + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 6a4fd7660..914b549fb 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -390,7 +390,8 @@ defmodule PlausibleWeb.Router do delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import - post "/:website/settings/export", SiteController, :export + post "/:website/settings/export", SiteController, :csv_export + get "/:website/settings/import", SiteController, :csv_import get "/:domain/export", StatsController, :csv_export get "/:domain/*path", StatsController, :stats diff --git a/lib/plausible_web/templates/site/csv_import.html.heex b/lib/plausible_web/templates/site/csv_import.html.heex new file mode 100644 index 000000000..ae9fc4038 --- /dev/null +++ b/lib/plausible_web/templates/site/csv_import.html.heex @@ -0,0 +1,20 @@ +
    +

    Import from CSV files

    + +
    +

    + Please ensure each file follows + <.styled_link href="https://plausible.io/docs/csv-import"> + our CSV format guidelines. + +

    + +

    + You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area. +

    +
    + + <%= live_render(@conn, PlausibleWeb.Live.CSVImport, + session: %{"site_id" => @site.id, "user_id" => @current_user.id} + ) %> +
    diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex index afc374c32..4011ac033 100644 --- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex +++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex @@ -20,9 +20,9 @@ New CSV import