From 73923404b07f07244bfee599d8c08fd376bf797e Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 2 Jan 2024 13:18:51 +0100 Subject: [PATCH 01/10] Improve approaching accept_traffic_until e-mail (#3660) * Improve approaching accept_traffic_until e-mail * fixup * format --- lib/plausible_web/email.ex | 2 +- .../email/approaching_accept_traffic_until.html.heex | 7 +++---- test/plausible_web/email_test.exs | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex index fd1c7300a..113ba25f1 100644 --- a/lib/plausible_web/email.ex +++ b/lib/plausible_web/email.ex @@ -358,7 +358,7 @@ defmodule PlausibleWeb.Email do |> to(user.email) |> tag("drop-traffic-warning-first") |> subject("We'll stop counting your stats") - |> render("approaching_accept_traffic_until.html", time: "next week") + |> render("approaching_accept_traffic_until.html", time: "next week", user: user) end def approaching_accept_traffic_until_tomorrow(user) do diff --git a/lib/plausible_web/templates/email/approaching_accept_traffic_until.html.heex b/lib/plausible_web/templates/email/approaching_accept_traffic_until.html.heex index f5e12414a..88e2f71e6 100644 --- a/lib/plausible_web/templates/email/approaching_accept_traffic_until.html.heex +++ b/lib/plausible_web/templates/email/approaching_accept_traffic_until.html.heex @@ -1,8 +1,7 @@ You used to have an active account with Plausible Analytics, a simple, lightweight, open source and privacy-first Google Analytics alternative. - +

We've noticed that you're still sending us stats so we're writing to inform you that we'll stop accepting stats from your sites <%= @time %>. We're an independent, bootstrapped service and we don't sell your data, so this will reduce our server costs and help keep us sustainable. - -If you'd like to continue counting your site stats in a privacy-friendly way, please +

If you'd like to continue counting your site stats in a privacy-friendly way, please login to your Plausible account and start a subscription. - +

Do you have any questions or need help with anything? Just reply to this email and we'll gladly help. diff --git a/test/plausible_web/email_test.exs b/test/plausible_web/email_test.exs index fc52753c4..9a23e0612 100644 --- a/test/plausible_web/email_test.exs +++ b/test/plausible_web/email_test.exs @@ -247,13 +247,14 @@ defmodule PlausibleWeb.EmailTest do describe "approaching accept_traffic_until" do test "renders first warning" do - user = build(:user) + user = build(:user, name: "John Doe") %{html_body: body, subject: subject} = PlausibleWeb.Email.approaching_accept_traffic_until(user) assert subject == "We'll stop counting your stats" assert body =~ plausible_link() + assert body =~ "Hey John," assert body =~ "We've noticed that you're still sending us stats so we're writing to inform you that we'll stop accepting stats from your sites next week." From 8be9397199d2f2b1ae424b198cd9e60f82446cdc Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 2 Jan 2024 13:19:04 +0100 Subject: [PATCH 02/10] Skip breakdown of imported pages with pageviews=0 (#3655) * Skip breakdown of imported pages with pageviews=0 * Fixup * Prove good impoted records are merged --- lib/plausible/stats/imported.ex | 1 + .../breakdown_test.exs | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 7e44b2005..394ba6fe8 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -368,6 +368,7 @@ defmodule Plausible.Stats.Imported do defp select_imported_metrics(q, [:pageviews | rest]) do q + |> where([i], i.pageviews > 0) |> select_merge([i], %{pageviews: sum(i.pageviews)}) |> select_imported_metrics(rest) end diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 3d9c9542e..eb603f158 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -523,6 +523,53 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do } end + test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed", + %{conn: conn, site: site} do + site = + site + |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok") + |> Plausible.Repo.update!() + + populate_stats(site, [ + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, + pathname: "/plausible.io", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:imported_pages, + page: "/skip-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 0 + ), + build(:imported_pages, + page: "/include-me", + date: ~D[2021-01-01], + visitors: 1, + pageviews: 1 + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:page", + "with_imported" => "true", + "metrics" => "pageviews" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/", "pageviews" => 2}, + %{"page" => "/plausible.io", "pageviews" => 1}, + %{"page" => "/include-me", "pageviews" => 1} + ] + } + end + test "breakdown by event:page", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]), From 1342135908d33fb44c9fd603ddadd8a97d9362e0 Mon Sep 17 00:00:00 2001 From: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Date: Tue, 2 Jan 2024 12:31:08 +0000 Subject: [PATCH 03/10] Add `custom_props.csv` back to CSV export for Growth plans (#3656) * restore special props access for growth plans * format * refactor if condition Co-authored-by: hq1 * format --------- Co-authored-by: hq1 --- .../controllers/api/stats_controller.ex | 35 ++++++++----- .../controllers/stats_controller.ex | 13 ++--- .../controllers/stats_controller_test.exs | 52 ++++++++++++++++--- 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index cccb5f253..849271675 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -1214,21 +1214,30 @@ defmodule PlausibleWeb.Api.StatsController do prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query) - values = - prop_names - |> Enum.map(fn prop_key -> - breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key)) - |> Enum.map(&Map.put(&1, :property, prop_key)) - |> transform_keys(%{:name => :value}) - end) - |> Enum.concat() + prop_names = + if Plausible.Billing.Feature.Props.enabled?(site) do + prop_names + else + prop_names |> Enum.filter(&(&1 in Plausible.Props.internal_keys())) + end - percent_or_cr = - if query.filters["event:goal"], - do: :conversion_rate, - else: :percentage + if not Enum.empty?(prop_names) do + values = + prop_names + |> Enum.map(fn prop_key -> + breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key)) + |> Enum.map(&Map.put(&1, :property, prop_key)) + |> transform_keys(%{:name => :value}) + end) + |> Enum.concat() + + percent_or_cr = + if query.filters["event:goal"], + do: :conversion_rate, + else: :percentage - to_csv(values, [:property, :value, :visitors, :events, percent_or_cr]) + to_csv(values, [:property, :value, :visitors, :events, percent_or_cr]) + end end defp breakdown_custom_prop_values(site, %{"prop_key" => prop_key} = params) do diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index b6bee8b11..e0e5fda39 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -159,18 +159,10 @@ defmodule PlausibleWeb.StatsController do ~c"operating_systems.csv" => fn -> Api.StatsController.operating_systems(conn, params) end, ~c"devices.csv" => fn -> Api.StatsController.screen_sizes(conn, params) end, ~c"conversions.csv" => fn -> Api.StatsController.conversions(conn, params) end, - ~c"referrers.csv" => fn -> Api.StatsController.referrers(conn, params) end + ~c"referrers.csv" => fn -> Api.StatsController.referrers(conn, params) end, + ~c"custom_props.csv" => fn -> Api.StatsController.all_custom_prop_values(conn, params) end } - csvs = - if Plausible.Billing.Feature.Props.enabled?(site) do - Map.put(csvs, ~c"custom_props.csv", fn -> - Api.StatsController.all_custom_prop_values(conn, params) - end) - else - csvs - end - csv_values = Map.values(csvs) |> Plausible.ClickhouseRepo.parallel_tasks() @@ -178,6 +170,7 @@ defmodule PlausibleWeb.StatsController do csvs = Map.keys(csvs) |> Enum.zip(csv_values) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) csvs = [{~c"visitors.csv", visitors} | csvs] diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index b33d87457..0cb547379 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -146,7 +146,6 @@ defmodule PlausibleWeb.StatsControllerTest do assert ~c"cities.csv" in zip assert ~c"conversions.csv" in zip assert ~c"countries.csv" in zip - assert ~c"custom_props.csv" in zip assert ~c"devices.csv" in zip assert ~c"entry_pages.csv" in zip assert ~c"exit_pages.csv" in zip @@ -161,15 +160,54 @@ defmodule PlausibleWeb.StatsControllerTest do assert ~c"utm_terms.csv" in zip end - test "does not export custom properties when site owner is on a growth plan", %{ + test "exports only internally used props in custom_props.csv for a growth plan", %{ conn: conn, - site: site, - user: user + site: site } do - insert(:growth_subscription, user: user) - response = conn |> get("/" <> site.domain <> "/export") |> response(200) + {:ok, site} = Plausible.Props.allow(site, ["author"]) + + site = Repo.preload(site, :owner) + insert(:growth_subscription, user: site.owner) + populate_stats(site, [ + build(:pageview, "meta.key": ["author"], "meta.value": ["a"]), + build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"]) + ]) + + conn = get(conn, "/" <> site.domain <> "/export?period=day") + assert response = response(conn, 200) {:ok, zip} = :zip.unzip(response, [:memory]) + + {_filename, result} = + Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end) + + assert parse_csv(result) == [ + ["property", "value", "visitors", "events", "percentage"], + ["url", "b", "1", "1", "50.0"], + ["url", "(none)", "1", "1", "50.0"], + [""] + ] + end + + test "does not include custom_props.csv for a growth plan if no internal props used", %{ + conn: conn, + site: site + } do + {:ok, site} = Plausible.Props.allow(site, ["author"]) + + site = Repo.preload(site, :owner) + insert(:growth_subscription, user: site.owner) + + populate_stats(site, [ + build(:pageview, "meta.key": ["author"], "meta.value": ["a"]) + ]) + + {:ok, zip} = + conn + |> get("/#{site.domain}/export?period=day") + |> response(200) + |> :zip.unzip([:memory]) + files = Map.new(zip) refute Map.has_key?(files, ~c"custom_props.csv") @@ -190,7 +228,7 @@ defmodule PlausibleWeb.StatsControllerTest do |> response(400) end - test "exports allowed event props", %{conn: conn, site: site} do + test "exports allowed event props for a trial account", %{conn: conn, site: site} do {:ok, site} = Plausible.Props.allow(site, ["author", "logged_in"]) populate_stats(site, [ From 7efa253e3f0c890cf330694e64438c9492265567 Mon Sep 17 00:00:00 2001 From: Marko Saric <34340819+metmarkosaric@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:27:25 +0100 Subject: [PATCH 04/10] Add a note about site settings being available on the locked screen (#3650) * Add a note about site settings being available on the locked screen some people are not aware that they can access site settings when the dashboard is locked. i've tried to add that to the note here as it should improve the user experience and minimize the number of people reaching out to support * Adjust markup a bit * Convert view to HEEx and apply predefined components --------- Co-authored-by: Adrian Gruntkowski --- .../templates/stats/site_locked.html.eex | 49 ------------ .../templates/stats/site_locked.html.heex | 80 +++++++++++++++++++ 2 files changed, 80 insertions(+), 49 deletions(-) delete mode 100644 lib/plausible_web/templates/stats/site_locked.html.eex create mode 100644 lib/plausible_web/templates/stats/site_locked.html.heex diff --git a/lib/plausible_web/templates/stats/site_locked.html.eex b/lib/plausible_web/templates/stats/site_locked.html.eex deleted file mode 100644 index 32140805a..000000000 --- a/lib/plausible_web/templates/stats/site_locked.html.eex +++ /dev/null @@ -1,49 +0,0 @@ -
-
-
-
- -
-

- Dashboard locked -

- - <%= case @conn.assigns[:current_user_role] do %> - <% :owner -> %> -
-

- This dashboard is locked because you don't have a valid subscription. We're still counting the stats but your access to the dashboard is restricted. Please subscribe to the appropriate tier with the link below to access the stats again. -

-
-
- <%= link("Manage my subscription", to: "/settings", class: "inline-flex items-center px-4 py-2 border border-transparent shadow-sm font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 sm:text-sm") %> -
- <% role when role in [:admin, :viewer] -> %> -
-

- This dashboard is currently locked and cannot be accessed. The site owner <%= @site.owner.email %> must upgrade their subscription plan in order to - unlock the stats. -

-
-

Want to pay for this site with the account you're logged in with?

-

Contact <%= @site.owner.email %> and ask them to <%= link("transfer the ownership", class: "text-indigo-500", to: "https://plausible.io/docs/transfer-ownership", rel: "noreferrer") %> of the site over to you

-
-
-
- <%= link("Back to my sites", to: "/sites", class: "inline-flex items-center px-4 py-2 border border-transparent shadow-sm font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 sm:text-sm") %> -
- <% _ -> %> -
-

- This dashboard is currently locked and cannot be accessed. You can check back later or contact the site owner to unlock it. -

-
- <%= if @conn.assigns[:current_user] do %> -
- <%= link("Back to my sites", to: "/sites", class: "inline-flex items-center px-4 py-2 border border-transparent shadow-sm font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 sm:text-sm") %> -
- <% end %> - <% end %> -
-
-
diff --git a/lib/plausible_web/templates/stats/site_locked.html.heex b/lib/plausible_web/templates/stats/site_locked.html.heex new file mode 100644 index 000000000..cf9327559 --- /dev/null +++ b/lib/plausible_web/templates/stats/site_locked.html.heex @@ -0,0 +1,80 @@ +
+
+
+
+ + + + +
+

+ Dashboard locked +

+ + <%= case @conn.assigns[:current_user_role] do %> + <% :owner -> %> +
+

+ This dashboard is locked because you don't have a valid subscription. + Please subscribe to the appropriate tier with the link below to access the stats again. +

+

+ You can check the + <.styled_link href={Routes.site_path(@conn, :settings_general, @site.domain)}> + site settings + + and we're still counting the stats but your access to the dashboard is restricted. +

+
+
+ <.button_link href={Routes.auth_path(@conn, :user_settings)}> + Manage my subscription + +
+ <% role when role in [:admin, :viewer] -> %> +
+

+ This dashboard is currently locked and cannot be accessed. The site owner + <%= @site.owner.email %> + must upgrade their subscription plan in order to + unlock the stats. +

+
+

Want to pay for this site with the account you're logged in with?

+

+ Contact <%= @site.owner.email %> and ask them to + <.styled_link href="https://plausible.io/docs/transfer-ownership" new_tab={true}> + transfer the ownership + + of the site over to you. +

+
+
+
+ <.button_link href={Routes.site_path(@conn, :index)}>Back to my sites +
+ <% _ -> %> +
+

+ This dashboard is currently locked and cannot be accessed. + You can check back later or contact the site owner to unlock the stats. +

+
+ <%= if @conn.assigns[:current_user] do %> +
+ <.button_link href={Routes.site_path(@conn, :index)}>Back to my sites +
+ <% end %> + <% end %> +
+
+
From 21bbd3835afbae984e1af6f16b1cec68e3d0a66b Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 2 Jan 2024 14:38:02 +0100 Subject: [PATCH 05/10] Disallow funny business on timezone entry (#3662) * Disallow funny business on timezone entry * Add external API test --- lib/plausible/site.ex | 11 +++++++++++ test/plausible/site/sites_test.exs | 9 +++++++++ .../api/external_sites_controller_test.exs | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex index c571619ab..789c3877e 100644 --- a/lib/plausible/site.ex +++ b/lib/plausible/site.ex @@ -68,6 +68,7 @@ defmodule Plausible.Site do |> cast(attrs, [:domain, :timezone]) |> clean_domain() |> validate_required([:domain, :timezone]) + |> validate_timezone() |> validate_domain_format() |> validate_domain_reserved_characters() |> unique_constraint(:domain, @@ -265,4 +266,14 @@ defmodule Plausible.Site do changeset end end + + defp validate_timezone(changeset) do + tz = get_field(changeset, :timezone) + + if Timex.is_valid_timezone?(tz) do + changeset + else + add_error(changeset, :timezone, "is invalid") + end + end end diff --git a/test/plausible/site/sites_test.exs b/test/plausible/site/sites_test.exs index bc70c1696..4898dbfcf 100644 --- a/test/plausible/site/sites_test.exs +++ b/test/plausible/site/sites_test.exs @@ -12,6 +12,15 @@ defmodule Plausible.SitesTest do assert {:ok, %{site: %{domain: "example.com", timezone: "Europe/London"}}} = Sites.create(user, params) end + + test "fails on invalid timezone" do + user = insert(:user) + + params = %{"domain" => "example.com", "timezone" => "blah"} + + assert {:error, :site, %{errors: [timezone: {"is invalid", []}]}, %{}} = + Sites.create(user, params) + end end describe "is_member?" do diff --git a/test/plausible_web/controllers/api/external_sites_controller_test.exs b/test/plausible_web/controllers/api/external_sites_controller_test.exs index 7bc8744d5..1eb9650d3 100644 --- a/test/plausible_web/controllers/api/external_sites_controller_test.exs +++ b/test/plausible_web/controllers/api/external_sites_controller_test.exs @@ -26,6 +26,18 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do } end + test "timezone is validated", %{conn: conn} do + conn = + post(conn, "/api/v1/sites", %{ + "domain" => "some-site.domain", + "timezone" => "d" + }) + + assert json_response(conn, 400) == %{ + "error" => "timezone: is invalid" + } + end + test "timezone defaults to Etc/UTC", %{conn: conn} do conn = post(conn, "/api/v1/sites", %{ From af9b34a1c6bd53f9f26b194fe3c970d72db9c83d Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 2 Jan 2024 16:25:50 +0100 Subject: [PATCH 06/10] Revert "Prevent scrollbar click to close modal (#3651)" (#3664) This reverts commit 75a7ef52410b7ffa8bc373eeb74f335d7a296918. --- assets/js/dashboard/stats/modals/modal.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/js/dashboard/stats/modals/modal.js b/assets/js/dashboard/stats/modals/modal.js index b30ec5140..4b031f360 100644 --- a/assets/js/dashboard/stats/modals/modal.js +++ b/assets/js/dashboard/stats/modals/modal.js @@ -24,7 +24,7 @@ class Modal extends React.Component { componentDidMount() { document.body.style.overflow = 'hidden'; document.body.style.height = '100vh'; - document.addEventListener("click", this.handleClickOutside); + document.addEventListener("mousedown", this.handleClickOutside); document.addEventListener("keyup", this.handleKeyup); window.addEventListener('resize', this.handleResize, false); this.handleResize(); @@ -33,7 +33,7 @@ class Modal extends React.Component { componentWillUnmount() { document.body.style.overflow = null; document.body.style.height = null; - document.removeEventListener("click", this.handleClickOutside); + document.removeEventListener("mousedown", this.handleClickOutside); document.removeEventListener("keyup", this.handleKeyup); window.removeEventListener('resize', this.handleResize, false); } From b6a2acb57a3015949d616cc5d1dce8bb4dbf9996 Mon Sep 17 00:00:00 2001 From: hq1 Date: Wed, 3 Jan 2024 11:19:25 +0100 Subject: [PATCH 07/10] Update accept_traffic_until notifications (#3665) * Update accept_traffic_until notifications: - extend has_stats to 48h - ensure the user is properly greeted * Clarify the structure passed to email template function --- lib/plausible_web/email.ex | 18 ++++++--- .../accept_traffic_until_notification.ex | 7 ++-- test/workers/accept_traffic_until_test.exs | 39 ++++++++++++++++++- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex index 113ba25f1..81aaff7e9 100644 --- a/lib/plausible_web/email.ex +++ b/lib/plausible_web/email.ex @@ -353,20 +353,26 @@ defmodule PlausibleWeb.Email do }) end - def approaching_accept_traffic_until(user) do + def approaching_accept_traffic_until(notification) do base_email() - |> to(user.email) + |> to(notification.email) |> tag("drop-traffic-warning-first") |> subject("We'll stop counting your stats") - |> render("approaching_accept_traffic_until.html", time: "next week", user: user) + |> render("approaching_accept_traffic_until.html", + time: "next week", + user: %{email: notification.email, name: notification.name} + ) end - def approaching_accept_traffic_until_tomorrow(user) do + def approaching_accept_traffic_until_tomorrow(notification) do base_email() - |> to(user.email) + |> to(notification.email) |> tag("drop-traffic-warning-final") |> subject("A reminder that we'll stop counting your stats tomorrow") - |> render("approaching_accept_traffic_until.html", time: "tomorrow") + |> render("approaching_accept_traffic_until.html", + time: "tomorrow", + user: %{email: notification.email, name: notification.name} + ) end @doc """ diff --git a/lib/workers/accept_traffic_until_notification.ex b/lib/workers/accept_traffic_until_notification.ex index 6ac7cbba3..7d6890996 100644 --- a/lib/workers/accept_traffic_until_notification.ex +++ b/lib/workers/accept_traffic_until_notification.ex @@ -40,7 +40,8 @@ defmodule Plausible.Workers.AcceptTrafficUntil do id: u.id, email: u.email, deadline: u.accept_traffic_until, - site_ids: fragment("array_agg(?.site_id)", sm) + site_ids: fragment("array_agg(?.site_id)", sm), + name: u.name }, group_by: u.id ) @@ -68,11 +69,11 @@ defmodule Plausible.Workers.AcceptTrafficUntil do end defp has_stats?(site_ids, today) do - yesterday = Date.add(today, -1) + ago_2d = Date.add(today, -2) ClickhouseRepo.exists?( from e in "events_v2", - where: fragment("toDate(?) >= ?", e.timestamp, ^yesterday), + where: fragment("toDate(?) >= ?", e.timestamp, ^ago_2d), where: e.site_id in ^site_ids ) end diff --git a/test/workers/accept_traffic_until_test.exs b/test/workers/accept_traffic_until_test.exs index b66b05908..af76df44b 100644 --- a/test/workers/accept_traffic_until_test.exs +++ b/test/workers/accept_traffic_until_test.exs @@ -12,6 +12,7 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do tomorrow = today |> Date.add(+1) user1 = insert(:user, accept_traffic_until: next_week) + user2 = insert(:user, accept_traffic_until: tomorrow) _site1 = insert(:site, members: [user1]) @@ -24,6 +25,37 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do refute_notifications(user2.email) end + test "does not send any notifications when site has stats older than 2d" do + today = Date.utc_today() + next_week = today |> Date.add(+7) + + user = insert(:user, accept_traffic_until: next_week) + + :site + |> insert(members: [user]) + |> populate_stats([build(:pageview, timestamp: Date.add(today, -3))]) + + {:ok, 1} = AcceptTrafficUntil.perform(nil) + + refute_notifications(user.email) + end + + test "does send notification when last stat is 2d old" do + today = Date.utc_today() + next_week = today |> Date.add(+7) + + user = + insert(:user, accept_traffic_until: next_week) + + :site + |> insert(members: [user]) + |> populate_stats([build(:pageview, timestamp: Date.add(today, -2))]) + + {:ok, 1} = AcceptTrafficUntil.perform(nil) + + assert_weekly_notification(user.email) + end + test "tomorrow: sends one e-mail" do tomorrow = Date.utc_today() |> Date.add(+1) user = insert(:user, accept_traffic_until: tomorrow) @@ -107,16 +139,19 @@ defmodule Plausible.Workers.AcceptTrafficUntilTest do defp assert_weekly_notification(email) when is_binary(email) do assert_email_delivered_with( + html_body: ~r/Hey Jane,/, to: [nil: email], - subject: PlausibleWeb.Email.approaching_accept_traffic_until(%{email: email}).subject + subject: + PlausibleWeb.Email.approaching_accept_traffic_until(%{name: "", email: email}).subject ) end defp assert_final_notification(email) when is_binary(email) do assert_email_delivered_with( + html_body: ~r/Hey Jane,/, to: [nil: email], subject: - PlausibleWeb.Email.approaching_accept_traffic_until_tomorrow(%{email: email}).subject + PlausibleWeb.Email.approaching_accept_traffic_until_tomorrow(%{name: "", email: email}).subject ) end From 7755d4fd293706fe78afd0a688e05b2bbdf618d6 Mon Sep 17 00:00:00 2001 From: hq1 Date: Wed, 3 Jan 2024 11:31:07 +0100 Subject: [PATCH 08/10] Lock traffic data backfill (#3648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP snapshot * Don't assume that subscription and trial can't co-exist 🤷 * Don't look up subscription status if payment is overdue --- ...20101920_backfill_accept_traffic_until.exs | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 priv/repo/migrations/20231220101920_backfill_accept_traffic_until.exs diff --git a/priv/repo/migrations/20231220101920_backfill_accept_traffic_until.exs b/priv/repo/migrations/20231220101920_backfill_accept_traffic_until.exs new file mode 100644 index 000000000..90fb71357 --- /dev/null +++ b/priv/repo/migrations/20231220101920_backfill_accept_traffic_until.exs @@ -0,0 +1,81 @@ +defmodule Plausible.Repo.Migrations.BackfillAcceptTrafficUntil do + use Ecto.Migration + + def change do + # trials that are about to expire get extra 14 days + # regardless of the effective end date, this still leaves a room for both notifications + execute """ + UPDATE users + SET accept_traffic_until = trial_expiry_date + 14 + WHERE + trial_expiry_date IS NOT NULL + AND + trial_expiry_date >= CURRENT_DATE + """ + + # free plans + execute """ + UPDATE users AS u + SET accept_traffic_until = '2135-01-01' + WHERE + EXISTS ( + SELECT 1 + FROM subscriptions s + WHERE + user_id = u.id + AND + paddle_plan_id = 'free_10k' + ) + """ + + # abandoned accounts (trial ended and no valid subscriptions) still get a random + # phase-out period so that both notifications can be delivered + execute """ + UPDATE users + SET accept_traffic_until = CURRENT_DATE + TRUNC(RANDOM() * (20 - 8 + 1) + 8)::int + WHERE + NOT EXISTS ( + SELECT 1 + FROM subscriptions + WHERE + subscriptions.user_id = users.id + ) + AND + trial_expiry_date IS NOT NULL + AND + trial_expiry_date < CURRENT_DATE + """ + + # all the non-free subscriptions + execute """ + UPDATE users u1 + SET accept_traffic_until = s.next_bill_date + 30 + FROM users u2 + INNER JOIN LATERAL ( + SELECT * FROM subscriptions sub WHERE u2.id = sub.user_id ORDER BY sub.inserted_at DESC LIMIT 1 + ) s ON (true) + WHERE + u1.id = u2.id + AND + s.user_id = u1.id + AND + s.paddle_plan_id != 'free_10k' + """ + + # subscription for which current period needs payment) + execute """ + UPDATE users u1 + SET accept_traffic_until = CURRENT_DATE + TRUNC(RANDOM() * (20 - 8 + 1) + 8)::int + FROM users u2 + INNER JOIN LATERAL ( + SELECT * FROM subscriptions sub WHERE u2.id = sub.user_id ORDER BY sub.inserted_at DESC LIMIT 1 + ) s ON (true) + WHERE + s.user_id = u1.id + AND + u1.id = u2.id + AND + s.next_bill_date < CURRENT_DATE + """ + end +end From 155a7a56a70655042a884d464dae5b89925a7093 Mon Sep 17 00:00:00 2001 From: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:31:52 +0000 Subject: [PATCH 09/10] Fix goal filter inconsistency (#3658) * remove unused code * add e.name == "pageview" condition to pageview goals This fixes the weird behavior where filtering by a pageview goal would also return custom events and vice versa. * update changelog --- CHANGELOG.md | 1 + lib/plausible/stats/base.ex | 42 ++-- lib/plausible/stats/breakdown.ex | 2 +- .../api/stats_controller/conversions_test.exs | 216 +++++++----------- 4 files changed, 99 insertions(+), 162 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 516e76ea9..e39dcba21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file. - Replace `CLICKHOUSE_MAX_BUFFER_SIZE` with `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` ### Fixed +- Stop returning custom events in goal breakdown with a pageview goal filter and vice versa - Only return `(none)` values in custom property breakdown for the first page (pagination) of results - Fixed weekly/monthly e-mail report [rendering issues](https://github.com/plausible/analytics/issues/284) - Fix [broken interval selection](https://github.com/plausible/analytics/issues/2982) in the all time view plausible/analytics#3110 diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 684d2676f..ae3e9bba9 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -60,18 +60,24 @@ defmodule Plausible.Stats.Base do q = case query.filters["event:goal"] do {:is, {:page, path}} -> - from(e in q, where: e.pathname == ^path) + from(e in q, where: e.pathname == ^path and e.name == "pageview") {:matches, {:page, expr}} -> regex = page_regex(expr) - from(e in q, where: fragment("match(?, ?)", e.pathname, ^regex)) + + from(e in q, + where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview" + ) {:is, {:event, event}} -> from(e in q, where: e.name == ^event) {:member, clauses} -> {events, pages} = split_goals(clauses) - from(e in q, where: e.pathname in ^pages or e.name in ^events) + + from(e in q, + where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events + ) {:matches_member, clauses} -> {events, pages} = split_goals(clauses, &page_regex/1) @@ -85,7 +91,10 @@ defmodule Plausible.Stats.Base do page_clause = if Enum.any?(pages) do - dynamic([x], fragment("multiMatchAny(?, ?)", x.pathname, ^pages)) + dynamic( + [x], + fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview" + ) else dynamic([x], false) end @@ -94,31 +103,6 @@ defmodule Plausible.Stats.Base do from(e in q, where: ^where_clause) - {:not_matches_member, clauses} -> - {events, pages} = split_goals(clauses, &page_regex/1) - - event_clause = - if Enum.any?(events) do - dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events)) - else - dynamic([x], false) - end - - page_clause = - if Enum.any?(pages) do - dynamic([x], fragment("multiMatchAny(?, ?)", x.pathname, ^pages)) - else - dynamic([x], false) - end - - where_clause = dynamic([], not (^event_clause or ^page_clause)) - - from(e in q, where: ^where_clause) - - {:not_member, clauses} -> - {events, pages} = split_goals(clauses) - from(e in q, where: e.pathname not in ^pages and e.name not in ^events) - nil -> q end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 634334e00..c3716041b 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -57,7 +57,7 @@ defmodule Plausible.Stats.Breakdown do "notEmpty(multiMatchAllIndices(?, ?) as indices)", e.pathname, ^page_regexes - ), + ) and e.name == "pageview", group_by: fragment("index"), select: %{ index: fragment("arrayJoin(indices) as index"), diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index e9620ca65..98538e3c0 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -265,16 +265,17 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do conn: conn, site: site } do - populate_stats(site, [ - build(:event, name: "Signup"), - build(:event, name: "Signup"), + [ build(:event, name: "Payment", pathname: "/checkout", revenue_reporting_amount: Decimal.new("10.00"), revenue_reporting_currency: "EUR" ) - ]) + ] + |> Enum.concat(build_list(2, :event, name: "Signup")) + |> Enum.concat(build_list(3, :pageview, pathname: "/checkout")) + |> then(fn events -> populate_stats(site, events) end) insert(:goal, %{site: site, event_name: "Signup"}) insert(:goal, %{site: site, page_path: "/checkout"}) @@ -286,7 +287,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert [ %{ "average_revenue" => %{"long" => "€10.00", "short" => "€10.0"}, - "conversion_rate" => 33.3, + "conversion_rate" => 16.7, "name" => "Payment", "events" => 1, "total_revenue" => %{"long" => "€10.00", "short" => "€10.0"}, @@ -294,7 +295,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do }, %{ "average_revenue" => nil, - "conversion_rate" => 66.7, + "conversion_rate" => 33.3, "name" => "Signup", "events" => 2, "total_revenue" => nil, @@ -302,11 +303,11 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do }, %{ "average_revenue" => nil, - "conversion_rate" => 33.3, + "conversion_rate" => 50.0, "name" => "Visit /checkout", - "events" => 1, + "events" => 3, "total_revenue" => nil, - "visitors" => 1 + "visitors" => 3 } ] == Enum.sort_by(response, & &1["name"]) end @@ -337,90 +338,110 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do describe "GET /api/stats/:domain/conversions - with goal filter" do setup [:create_user, :log_in, :create_new_site] - test "returns only the conversion that is filtered for", %{conn: conn, site: site} do + test "does not consider custom event pathname as a pageview goal completion", %{ + conn: conn, + site: site + } do populate_stats(site, [ - build(:pageview, pathname: "/"), build(:pageview, pathname: "/"), build(:pageview, pathname: "/register"), - build(:pageview, pathname: "/register"), - build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["A"]), - build(:event, name: "Signup", "meta.key": ["variant"], "meta.value": ["B"]) + build(:event, pathname: "/register", name: "Signup") ]) insert(:goal, %{site: site, page_path: "/register"}) insert(:goal, %{site: site, event_name: "Signup"}) - filters = Jason.encode!(%{goal: "Signup"}) + get_with_filter = fn filters -> + path = "/api/stats/#{site.domain}/conversions" + query = "?period=day&filters=#{Jason.encode!(filters)}" - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}" - ) + get(conn, path <> query) + |> json_response(200) + end - assert json_response(conn, 200) == [ - %{ - "name" => "Signup", - "visitors" => 2, - "events" => 2, - "conversion_rate" => 33.3 - } - ] + expected = [ + %{ + "name" => "Signup", + "visitors" => 1, + "events" => 1, + "conversion_rate" => 33.3 + } + ] + + # {:is, {:event, event}} filter type + assert get_with_filter.(%{goal: "Signup"}) == expected + + # {:member, clauses} filter type + assert get_with_filter.(%{goal: "Signup|Whatever"}) == expected + + # {:matches_member, clauses} filter type + assert get_with_filter.(%{goal: "Signup|Visit /whatever*"}) == expected end - test "can filter by multiple mixed goals", %{conn: conn, site: site} do + test "does not return custom events with the filtered pageview goal pathname", %{ + conn: conn, + site: site + } do populate_stats(site, [ build(:pageview, pathname: "/"), - build(:pageview, pathname: "/"), - build(:pageview, pathname: "/another"), build(:pageview, pathname: "/register"), - build(:event, name: "Signup"), - build(:event, name: "Signup") + build(:event, pathname: "/register", name: "Signup") ]) insert(:goal, %{site: site, page_path: "/register"}) + insert(:goal, %{site: site, page_path: "/regi**"}) insert(:goal, %{site: site, event_name: "Signup"}) - filters = Jason.encode!(%{goal: "Signup|Visit /register"}) + get_with_filter = fn filters -> + path = "/api/stats/#{site.domain}/conversions" + query = "?period=day&filters=#{Jason.encode!(filters)}" - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "Signup", - "visitors" => 2, - "events" => 2, - "conversion_rate" => 33.3 - }, - %{ - "name" => "Visit /register", - "visitors" => 1, - "events" => 1, - "conversion_rate" => 16.7 - } - ] + get(conn, path <> query) + |> json_response(200) + end + + expected = [ + %{ + "name" => "Visit /register", + "visitors" => 1, + "events" => 1, + "conversion_rate" => 33.3 + }, + %{ + "name" => "Visit /regi**", + "visitors" => 1, + "events" => 1, + "conversion_rate" => 33.3 + } + ] + + # {:is, {:page, path}} filter type + assert get_with_filter.(%{goal: "Visit+/register"}) == expected + + # {:matches, {:page, expr}} filter type + assert get_with_filter.(%{goal: "Visit+/regi**"}) == expected + + # {:member, clauses} filter type + assert get_with_filter.(%{goal: "Visit+/register|Whatever"}) == expected + + # {:matches_member, clauses} filter type + assert get_with_filter.(%{goal: "Visit+/register|Visit+/regi**"}) == expected end - test "can filter by multiple negated mixed goals", %{conn: conn, site: site} do + test "can filter by multiple mixed goals", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/"), build(:pageview, pathname: "/"), build(:pageview, pathname: "/another"), build(:pageview, pathname: "/register"), - build(:event, name: "CTA"), + build(:event, name: "Signup"), build(:event, name: "Signup") ]) insert(:goal, %{site: site, page_path: "/register"}) - insert(:goal, %{site: site, page_path: "/another"}) - insert(:goal, %{site: site, event_name: "CTA"}) insert(:goal, %{site: site, event_name: "Signup"}) - filters = Jason.encode!(%{goal: "!Signup|Visit /another"}) + filters = Jason.encode!(%{goal: "Signup|Visit /register"}) conn = get( @@ -430,10 +451,10 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do assert json_response(conn, 200) == [ %{ - "name" => "CTA", - "visitors" => 1, - "events" => 1, - "conversion_rate" => 16.7 + "name" => "Signup", + "visitors" => 2, + "events" => 2, + "conversion_rate" => 33.3 }, %{ "name" => "Visit /register", @@ -515,75 +536,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do } ] end - - test "can filter by not_matches_member filter type on goals", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, pathname: "/another"), - build(:pageview, pathname: "/another"), - build(:pageview, pathname: "/blog/post-1"), - build(:pageview, pathname: "/blog/post-2"), - build(:event, name: "CTA"), - build(:event, name: "Signup") - ]) - - insert(:goal, %{site: site, page_path: "/blog**"}) - insert(:goal, %{site: site, page_path: "/ano**"}) - insert(:goal, %{site: site, event_name: "CTA"}) - insert(:goal, %{site: site, event_name: "Signup"}) - - filters = Jason.encode!(%{goal: "!Signup|Visit /blog**"}) - - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "Visit /ano**", - "visitors" => 2, - "events" => 2, - "conversion_rate" => 33.3 - }, - %{ - "name" => "CTA", - "visitors" => 1, - "events" => 1, - "conversion_rate" => 16.7 - } - ] - end - - test "can combine wildcard and no wildcard in not_matches_member", %{conn: conn, site: site} do - populate_stats(site, [ - build(:pageview, pathname: "/blog/post-1"), - build(:pageview, pathname: "/blog/post-2"), - build(:pageview, pathname: "/billing/upgrade"), - build(:pageview, pathname: "/register") - ]) - - insert(:goal, %{site: site, page_path: "/blog/**"}) - insert(:goal, %{site: site, page_path: "/billing/upgrade"}) - insert(:goal, %{site: site, page_path: "/register"}) - - filters = Jason.encode!(%{goal: "!Visit /blog/**|Visit /billing/upgrade"}) - - conn = - get( - conn, - "/api/stats/#{site.domain}/conversions?period=day&filters=#{filters}" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "Visit /register", - "visitors" => 1, - "events" => 1, - "conversion_rate" => 25 - } - ] - end end describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do From 9e5d63ed96a7b1e49413f491435edd4a4eaf623c Mon Sep 17 00:00:00 2001 From: Vinicius Brasil Date: Wed, 3 Jan 2024 08:32:21 -0300 Subject: [PATCH 10/10] Display premium features tabs on the dashboard (#3646) * Display premium features tabs on the dashboard This commit makes the funnels and props tab on the dashboard visible even when the site owner's plan does not have access to that feature. This is to raise awareness of those features, and if the site owner does not want to see that tab, they can still click "Hide this report" to hide it. Previously, when the plan did not support a feature, the feature module `enabled/1` function returned `false`, regardless of the `sites.feature_enabled` toggle. This commit creates a new function called `opted_out/1` to differentiate access from explicitly opting out a feature. * Remove unused data-conversions-enabled attribute * DRY Plausible.Billing.Feature.check_availability/1 function * Allow opting out features the user doesn't have access to * add upgrade CTA to FeatureSetupNotice * fix JS linting errors * simplify notice.js * fix behaviour when deleting funnels saved to localstorage In case some other funnel exists, we will use that as the default selected one. If not, a feature setup notice will be displayed again. --------- Co-authored-by: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> Co-authored-by: Robert Joonas --- assets/js/dashboard.js | 8 +- assets/js/dashboard/components/notice.js | 14 ++-- assets/js/dashboard/stats/behaviours/index.js | 73 ++++++++++++++----- lib/plausible/billing/feature.ex | 46 +++++++----- .../controllers/stats_controller.ex | 2 + .../templates/stats/stats.html.eex | 8 +- test/plausible/billing/feature_test.exs | 19 ++++- 7 files changed, 120 insertions(+), 50 deletions(-) diff --git a/assets/js/dashboard.js b/assets/js/dashboard.js index e4e2fbcc7..c733c0905 100644 --- a/assets/js/dashboard.js +++ b/assets/js/dashboard.js @@ -16,10 +16,12 @@ if (container) { domain: container.dataset.domain, offset: container.dataset.offset, hasGoals: container.dataset.hasGoals === 'true', - conversionsEnabled: container.dataset.conversionsEnabled === 'true', - funnelsEnabled: container.dataset.funnelsEnabled === 'true', - propsEnabled: container.dataset.propsEnabled === 'true', hasProps: container.dataset.hasProps === 'true', + funnelsAvailable: container.dataset.funnelsAvailable === 'true', + propsAvailable: container.dataset.propsAvailable === 'true', + conversionsOptedOut: container.dataset.conversionsOptedOut === 'true', + funnelsOptedOut: container.dataset.funnelsOptedOut === 'true', + propsOptedOut: container.dataset.propsOptedOut === 'true', funnels: JSON.parse(container.dataset.funnels), statsBegin: container.dataset.statsBegin, nativeStatsBegin: container.dataset.nativeStatsBegin, diff --git a/assets/js/dashboard/components/notice.js b/assets/js/dashboard/components/notice.js index fd2d9876a..656c7abbe 100644 --- a/assets/js/dashboard/components/notice.js +++ b/assets/js/dashboard/components/notice.js @@ -2,7 +2,7 @@ import React from "react" import { sectionTitles } from "../stats/behaviours" import * as api from '../api' -export function FeatureSetupNotice({ site, feature, shortFeatureName, title, info, settingsLink, onHideAction }) { +export function FeatureSetupNotice({ site, feature, title, info, callToAction, onHideAction }) { const sectionTitle = sectionTitles[feature] const requestHideSection = () => { @@ -14,10 +14,10 @@ export function FeatureSetupNotice({ site, feature, shortFeatureName, title, inf } } - function setupButton() { + function renderCallToAction() { return ( - -

Set up {shortFeatureName}

+
+

{callToAction.action}

@@ -25,7 +25,7 @@ export function FeatureSetupNotice({ site, feature, shortFeatureName, title, inf ) } - function hideButton() { + function renderHideButton() { return (