From 5f42141e0833ec6953927c752861b91b053bc621 Mon Sep 17 00:00:00 2001 From: Vignesh Joglekar Date: Wed, 19 May 2021 02:21:43 -0500 Subject: [PATCH 01/15] Adds time on page as a top stat when a page filter is active (#1051) * Adds time on page as a top stat when a page filter is active * Removes dangling debug statement --- lib/plausible/stats/clickhouse.ex | 7 ++- .../controllers/api/stats_controller.ex | 48 ++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index a525799dd..e636af7cd 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -691,7 +691,7 @@ defmodule Plausible.Stats.Clickhouse do |> Enum.into(%{}) end - defp page_times_by_page_url(site, query, page_list) do + def page_times_by_page_url(site, query, page_list) do q = from( e in base_query_w_sessions(site, %Query{ @@ -725,7 +725,10 @@ defmodule Plausible.Stats.Clickhouse do FROM (#{base_query_raw})) WHERE s=s2 AND p IN tuple(?) GROUP BY p,p2,s) - GROUP BY p" |> ClickhouseRepo.query(base_query_raw_params ++ [page_list ++ ["/"]]) + GROUP BY p" + |> ClickhouseRepo.query( + base_query_raw_params ++ [(Enum.count(page_list) > 0 && page_list) || ["/"]] + ) end defp add_percentages(stat_list) do diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 5985e4689..33d7f510f 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -104,6 +104,51 @@ defmodule PlausibleWeb.Api.StatsController do } end + time_on_page = + if query.filters["page"] do + [{success, duration}, {prev_success, prev_duration}] = + Task.yield_many( + [ + Task.async(fn -> + {:ok, page_times} = + Stats.page_times_by_page_url(site, query, [query.filters["page"]]) + + page_times + end), + Task.async(fn -> + {:ok, page_times} = + Stats.page_times_by_page_url(site, prev_query, [query.filters["page"]]) + + page_times + end) + ], + 5000 + ) + |> Enum.map(fn {task, response} -> + case response do + nil -> + Task.shutdown(task, :brutal_kill) + {nil, nil} + + {:ok, page_times} -> + result = Enum.at(page_times.rows, 0) + result = if result, do: Enum.at(result, 1), else: nil + if result, do: {:ok, round(result)}, else: {:ok, 0} + + _ -> + response + end + end) + + if success == :ok && prev_success == :ok do + %{ + name: "Time on Page", + duration: duration, + change: percent_change(prev_duration, duration) + } + end + end + stats = [ %{ @@ -117,7 +162,8 @@ defmodule PlausibleWeb.Api.StatsController do change: percent_change(prev_pageviews, pageviews) }, %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate}, - visit_duration + visit_duration, + time_on_page ] |> Enum.filter(& &1) From 7153b366e52f3784ec4cbfeae0d059a34cdb9a05 Mon Sep 17 00:00:00 2001 From: Ru Singh Date: Wed, 19 May 2021 19:19:39 +0530 Subject: [PATCH 02/15] chore(docker): improve repeat contributions workflow (#1057) * This change adds two new commands to gracefully stop and remove the Postgres and Clickhouse docker containers. To do so, it also gives them a recognizable name. * Additionally, the Postgres container is updated to use a named volume for its data. This lower friction for repeat contributions where one would otherwise sign up and activate their accounts again and again each time. --- CONTRIBUTING.md | 7 +++++++ Makefile | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f96b4fd6..1435daf2b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,3 +23,10 @@ Make sure Docker, Elixir, Erlang and Node.js are all installed on your developme 3. Fill in the rest of the forms and for the domain use `dummy.site` 4. Run `make dummy_event` from the terminal to generate a fake pageview event for the dummy site. 5. You should now be all set! + +### Stopping Docker containers + +1. Stop and remove the Postgres container with `make postgres-stop`. +2. Stop and remove the Clickhouse container with `make clickhouse-stop`. + +Volumes are preserved. You'll find that the Postgres and Clickhouse state are retained when you bring them up again the next time: no need to re-register and so on. \ No newline at end of file diff --git a/Makefile b/Makefile index e669b6326..5e7c5c94e 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,14 @@ clickhouse: - docker run --detach -p 8123:8123 --ulimit nofile=262144:262144 --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse yandex/clickhouse-server:21.3.2.5 + docker run --detach -p 8123:8123 --ulimit nofile=262144:262144 --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse --name plausible_clickhouse yandex/clickhouse-server:21.3.2.5 + +clickhouse-stop: + docker stop plausible_clickhouse && docker rm plausible_clickhouse postgres: - docker run --detach -e POSTGRES_PASSWORD="postgres" -p 5432:5432 postgres:12 + docker run --detach -e POSTGRES_PASSWORD="postgres" -p 5432:5432 --volume=plausible_db:/var/lib/postgresql/data --name plausible_db postgres:12 + +postgres-stop: + docker stop plausible_db && docker rm plausible_db dummy_event: curl 'http://localhost:8000/api/event' \ From 9d700a059c4b9ff110312ea261e116782fe3cce1 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 19 May 2021 16:53:08 +0300 Subject: [PATCH 03/15] Include testing script as well --- lib/plausible_web/templates/layout/_tracking.html.eex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/plausible_web/templates/layout/_tracking.html.eex b/lib/plausible_web/templates/layout/_tracking.html.eex index db79f3604..519afe864 100644 --- a/lib/plausible_web/templates/layout/_tracking.html.eex +++ b/lib/plausible_web/templates/layout/_tracking.html.eex @@ -1,4 +1,5 @@ <%= if !Application.get_env(:plausible, :is_selfhost) && !@conn.assigns[:skip_plausible_tracking] do %> - + + <% end %> From c55450c8d71988b6f4ba74c63014b408d9ed2d22 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 20 May 2021 17:25:57 +0300 Subject: [PATCH 04/15] Include new testing script --- lib/plausible_web/templates/layout/_tracking.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible_web/templates/layout/_tracking.html.eex b/lib/plausible_web/templates/layout/_tracking.html.eex index 519afe864..e4bd69cd1 100644 --- a/lib/plausible_web/templates/layout/_tracking.html.eex +++ b/lib/plausible_web/templates/layout/_tracking.html.eex @@ -1,5 +1,5 @@ <%= if !Application.get_env(:plausible, :is_selfhost) && !@conn.assigns[:skip_plausible_tracking] do %> - + <% end %> From f203eba7c0e8134e55b38d6c632d7c96594cafdc Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 21 May 2021 11:30:59 +0300 Subject: [PATCH 05/15] Deprecate custom domains --- .../templates/email/welcome_email.html.eex | 10 ++++----- .../site/settings_custom_domain.html.eex | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/plausible_web/templates/email/welcome_email.html.eex b/lib/plausible_web/templates/email/welcome_email.html.eex index 419bb0f28..602eb71c0 100644 --- a/lib/plausible_web/templates/email/welcome_email.html.eex +++ b/lib/plausible_web/templates/email/welcome_email.html.eex @@ -5,11 +5,11 @@ We're super excited to have you on board!

Here are five steps to get the most out of your Plausible experience:

-* Serve the script as a <%= link("first party connection from your custom domain", to: "https://docs.plausible.io/custom-domain") %>
-* <%= link("Enable email reports", to: "https://docs.plausible.io/email-reports/") %> and notifications for <%= link("traffic spikes", to: "https://docs.plausible.io/traffic-spikes/") %>
-* <%= link("Integrate with Search Console", to: "https://docs.plausible.io/google-search-console-integration/") %> to get keyword phrases people find your site with
-* Set up some easy goals including <%= link("404 error pages", to: "https://docs.plausible.io/404-error-pages-tracking") %> and <%= link("outbound link clicks", to: "https://docs.plausible.io/outbound-link-click-tracking/") %>
-* <%= link("Opt out from counting your own visits", to: "https://docs.plausible.io/excluding/") %>
+* <%= link("Enable email reports", to: "https://docs.plausible.io/email-reports/") %> and notifications for <%= link("traffic spikes", to: "https://plausible.io/docs/traffic-spikes") %>
+* <%= link("Integrate with Search Console", to: "https://plausible.io/docs/google-search-console-integration") %> to get keyword phrases people find your site with
+* Set up some easy goals including <%= link("404 error pages", to: "https://plausible.io/docs/404-error-pages-tracking") %> and <%= link("outbound link clicks", to: "https://plausible.io/docs/outbound-link-click-tracking/") %>
+* <%= link("Opt out from counting your own visits", to: "https://plausible.io/docs/excluding") %>
+* If you're concerned about adblockers, <%= link("set up a proxy to bypass them", to: "https://plausible.io/docs/proxy/introduction") %>


Then you're ready to start exploring your fast loading, ethical and actionable <%= link("Plausible dashboard", to: "https://plausible.io/sites") %>.

diff --git a/lib/plausible_web/templates/site/settings_custom_domain.html.eex b/lib/plausible_web/templates/site/settings_custom_domain.html.eex index 42e7a4fa1..71eb3928f 100644 --- a/lib/plausible_web/templates/site/settings_custom_domain.html.eex +++ b/lib/plausible_web/templates/site/settings_custom_domain.html.eex @@ -1,3 +1,24 @@ +
+
+
+ +
+
+

+ Deprecated feature +

+
+

+ We are moving away from CNAME-based custom domains. If you're concerned about adblockers, we recommend + <%= link("setting up a proxy", class: "underline text-yellow-800 dark:text-yellow-400", to: "https://plausible.io/docs/proxy/introduction", target: "_blank") %> for your analytics script instead. +

+
+
+
+
+

Custom domain

From ff9f95ea18f30d5355201d46a9478f1346cf0b8c Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 21 May 2021 11:36:35 +0300 Subject: [PATCH 06/15] Fix old script --- tracker/src/p.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tracker/src/p.js b/tracker/src/p.js index 55edddcaa..c675bc409 100644 --- a/tracker/src/p.js +++ b/tracker/src/p.js @@ -2,9 +2,11 @@ // with some early customers. This script uses a cookie but this was an old version of Plausible. // Current script can be found in the tracker/src/plausible.js file -(function(window, plausibleHost){ +(function(){ 'use strict'; + var plausibleHost = new URL(scriptEl.src).origin + function setCookie(name,value) { var date = new Date(); date.setTime(date.getTime() + (3*365*24*60*60*1000)); // 3 YEARS @@ -117,4 +119,4 @@ } catch (e) { new Image().src = plausibleHost + '/api/error?message=' + encodeURIComponent(e.message); } -})(window, '<%= base_url %>'); +})(); From 5ee3167786f6a489a2a02eb18529e06f42eb0910 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 21 May 2021 12:06:12 +0300 Subject: [PATCH 07/15] Add special plan --- lib/plausible/billing/plans.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/plausible/billing/plans.ex b/lib/plausible/billing/plans.ex index f367247a1..7cc0895a4 100644 --- a/lib/plausible/billing/plans.ex +++ b/lib/plausible/billing/plans.ex @@ -3,6 +3,10 @@ defmodule Plausible.Billing.Plans do %{limit: 150_000_000, yearly_product_id: "648089", yearly_cost: "$4800"} ] + @unlisted_plans_v2 [ + %{limit: 10_000_000, monthly_product_id: "655350", yearly_cost: "$250"} + ] + @v2_pricing_date ~D[2021-05-13] def plans_for(user) do @@ -70,7 +74,7 @@ defmodule Plausible.Billing.Plans do end defp all_plans() do - plans_v1() ++ @unlisted_plans_v1 ++ plans_v2() + plans_v1() ++ @unlisted_plans_v1 ++ plans_v2() ++ @unlisted_plans_v2 end defp plans_v1() do From 2cf0c1a4561212134cfe8a6dd24e4f55db7a87f1 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 21 May 2021 14:57:00 +0300 Subject: [PATCH 08/15] Use `forwarded` header when X-Forwarded-For is missing --- config/test.exs | 5 ++- lib/plausible_web/remote_ip.ex | 9 +++++ .../api/external_controller_test.exs | 39 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 3d3b6d9a7..5ea4ce0e7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -23,7 +23,10 @@ config :geolix, %{ id: :country, adapter: Geolix.Adapter.Fake, - data: %{{1, 1, 1, 1} => %{country: %{iso_code: "US"}}} + data: %{ + {1, 1, 1, 1} => %{country: %{iso_code: "US"}}, + {1, 1, 1, 1, 1, 1, 1, 1} => %{country: %{iso_code: "US"}} + } } ] diff --git a/lib/plausible_web/remote_ip.ex b/lib/plausible_web/remote_ip.ex index e8ad3792b..fd7dcf816 100644 --- a/lib/plausible_web/remote_ip.ex +++ b/lib/plausible_web/remote_ip.ex @@ -2,6 +2,7 @@ defmodule PlausibleWeb.RemoteIp do def get(conn) do cf_connecting_ip = List.first(Plug.Conn.get_req_header(conn, "cf-connecting-ip")) forwarded_for = List.first(Plug.Conn.get_req_header(conn, "x-forwarded-for")) + forwarded = List.first(Plug.Conn.get_req_header(conn, "forwarded")) cond do cf_connecting_ip -> @@ -12,6 +13,14 @@ defmodule PlausibleWeb.RemoteIp do |> Enum.map(&String.trim/1) |> List.first() + forwarded -> + Regex.named_captures(~r/for=(?[^;,]+).*$/, forwarded) + |> Map.get("for") + # IPv6 addresses are enclosed in quote marks and square brackets: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded + |> String.trim("\"") + |> String.trim_leading("[") + |> String.trim_trailing("]") + true -> to_string(:inet_parse.ntoa(conn.remote_ip)) end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index 014067cd9..4c28d0694 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -510,6 +510,45 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview.country_code == "US" end + test "Uses the Forwarded header when cf-connecting-ip and x-forwarded-for are missing", %{ + conn: conn + } do + params = %{ + name: "pageview", + domain: "external-controller-test-forwarded.com", + url: "http://gigride.live/" + } + + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("forwarded", "by=0.0.0.0;for=1.1.1.1;host=somehost.com;proto=https") + |> post("/api/event", Jason.encode!(params)) + + pageview = get_event("external-controller-test-forwarded.com") + + assert pageview.country_code == "US" + end + + test "Forwarded header can parse ipv6", %{conn: conn} do + params = %{ + name: "pageview", + domain: "external-controller-test-forwarded-ipv6.com", + url: "http://gigride.live/" + } + + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header( + "forwarded", + "by=0.0.0.0;for=\"[1:1:1:1:1:1:1:1]\",for=0.0.0.0;host=somehost.com;proto=https" + ) + |> post("/api/event", Jason.encode!(params)) + + pageview = get_event("external-controller-test-forwarded-ipv6.com") + + assert pageview.country_code == "US" + end + test "URL is decoded", %{conn: conn} do params = %{ name: "pageview", From edc77236c708ca248f0c9b2139bf7e2a49dbf736 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 21 May 2021 16:01:52 +0300 Subject: [PATCH 09/15] Some improvements to pagination in dark mode --- assets/css/app.css | 2 +- lib/plausible_web/templates/site/index.html.eex | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assets/css/app.css b/assets/css/app.css index b68003678..58376b8b2 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -308,5 +308,5 @@ iframe[hidden] { } .pagination-link[disabled] { - @apply cursor-default bg-gray-100 pointer-events-none; + @apply cursor-default bg-gray-100 dark:bg-gray-300 pointer-events-none; } diff --git a/lib/plausible_web/templates/site/index.html.eex b/lib/plausible_web/templates/site/index.html.eex index 192ed4588..dc5e9367c 100644 --- a/lib/plausible_web/templates/site/index.html.eex +++ b/lib/plausible_web/templates/site/index.html.eex @@ -40,9 +40,9 @@ <%= if @pagination.total_pages > 1 do %> <%= pagination @conn, @pagination, [current_class: "is-current"], fn p -> %> -