From fea9bb32ee3ba19185026202f409721ab1a28241 Mon Sep 17 00:00:00 2001 From: hq1 Date: Tue, 11 Jul 2023 10:47:03 +0200 Subject: [PATCH] Prevent robots from looking too much into public dashboards (#3118) * Add noindex,nofollow to dashboard pages * Implement NoRobots plug * Enable NoRobots plug in the router * Fixup internal route * Fix double slash in the router * Add special bot treatment to plausible.io live demo page * Revert aggressive protection with agent detection --- .../controllers/api/internal_controller.ex | 10 +++--- .../controllers/stats_controller.ex | 12 +++++-- lib/plausible_web/plugs/no_robots.ex | 29 +++++++++++++++ lib/plausible_web/router.ex | 10 ++++-- .../templates/layout/app.html.eex | 1 + .../templates/layout/focus.html.eex | 1 + .../api/internal_controller_test.exs | 15 ++++++++ .../controllers/stats_controller_test.exs | 27 +++++++++++++- test/plausible_web/plugs/no_robots_test.exs | 36 +++++++++++++++++++ 9 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 lib/plausible_web/plugs/no_robots.ex create mode 100644 test/plausible_web/plugs/no_robots_test.exs diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.ex index b721bc9d0..c1e270751 100644 --- a/lib/plausible_web/controllers/api/internal_controller.ex +++ b/lib/plausible_web/controllers/api/internal_controller.ex @@ -6,12 +6,14 @@ defmodule PlausibleWeb.Api.InternalController do alias Plausible.Auth.User def domain_status(conn, %{"domain" => domain}) do - site = Sites.get_by_domain(domain) - - if Stats.has_pageviews?(site) do + with %User{id: user_id} <- conn.assigns[:current_user], + %Site{} = site <- Sites.get_by_domain(domain), + true <- Sites.has_admin_access?(user_id, site) || Auth.is_super_admin?(user_id), + true <- Stats.has_pageviews?(site) do json(conn, "READY") else - json(conn, "WAITING") + _ -> + json(conn, "WAITING") end end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 72b6ba9a4..991a62c46 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -67,7 +67,7 @@ defmodule PlausibleWeb.StatsController do funnels: Plausible.Funnels.list(site), stats_start_date: stats_start_date, native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at), - title: "Plausible · " <> site.domain, + title: title(conn, site), offer_email_report: offer_email_report, demo: demo, flags: get_flags(conn.assigns[:current_user]), @@ -295,7 +295,7 @@ defmodule PlausibleWeb.StatsController do funnels: Plausible.Funnels.list(shared_link.site), stats_start_date: shared_link.site.stats_start_date, native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at), - title: "Plausible · " <> shared_link.site.domain, + title: title(conn, shared_link.site), offer_email_report: false, demo: false, skip_plausible_tracking: true, @@ -343,4 +343,12 @@ defmodule PlausibleWeb.StatsController do !!is_or_nil end + + defp title(%{path_info: ["plausible.io"]}, _) do + "Plausible Analytics: Live Demo" + end + + defp title(_conn, site) do + "Plausible · " <> site.domain + end end diff --git a/lib/plausible_web/plugs/no_robots.ex b/lib/plausible_web/plugs/no_robots.ex new file mode 100644 index 000000000..86f178fbd --- /dev/null +++ b/lib/plausible_web/plugs/no_robots.ex @@ -0,0 +1,29 @@ +defmodule PlausibleWeb.Plugs.NoRobots do + @moduledoc """ + Rejects bot requests by any means available. + + We're adding `x-robots-tag` to the response header and annotate the conn + with "noindex, nofollow" under `private.robots` key. + + The only exception is, if the request is trying to access our live demo + at plausible.io/plausible.io - in which case we'll allow indexing, but deny + following links and skip the bot detection, in kind robots we trust. + """ + @behaviour Plug + import Plug.Conn + + @impl true + def init(opts), do: opts + + @impl true + def call(conn, _opts \\ nil) do + conn = + if conn.path_info == ["plausible.io"] do + put_private(conn, :robots, "index, nofollow") + else + put_private(conn, :robots, "noindex, nofollow") + end + + put_resp_header(conn, "x-robots-tag", conn.private.robots) + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 018ab227f..8c26db6f9 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -8,6 +8,7 @@ defmodule PlausibleWeb.Router do plug :fetch_session plug :fetch_live_flash plug :put_secure_browser_headers + plug PlausibleWeb.Plugs.NoRobots plug PlausibleWeb.FirstLaunchPlug, redirect_to: "/register" plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_seconds plug PlausibleWeb.AuthPlug @@ -17,6 +18,7 @@ defmodule PlausibleWeb.Router do pipeline :shared_link do plug :accepts, ["html"] plug :put_secure_browser_headers + plug PlausibleWeb.Plugs.NoRobots end pipeline :csrf do @@ -33,6 +35,7 @@ defmodule PlausibleWeb.Router do plug :accepts, ["json"] plug :fetch_session plug PlausibleWeb.AuthorizeSiteAccess + plug PlausibleWeb.Plugs.NoRobots end pipeline :public_api do @@ -42,6 +45,7 @@ defmodule PlausibleWeb.Router do pipeline :flags do plug :accepts, ["html"] plug :put_secure_browser_headers + plug PlausibleWeb.Plugs.NoRobots plug :fetch_session plug PlausibleWeb.CRMAuthPlug end @@ -50,7 +54,9 @@ defmodule PlausibleWeb.Router do forward "/sent-emails", Bamboo.SentEmailViewerPlug end - use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug] + use Kaffy.Routes, + scope: "/crm", + pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug] scope path: "/flags" do pipe_through :flags @@ -59,7 +65,7 @@ defmodule PlausibleWeb.Router do scope "/api/stats", PlausibleWeb.Api do pipe_through :internal_stats_api - get "/:domain//funnels/:id", StatsController, :funnel + get "/:domain/funnels/:id", StatsController, :funnel get "/:domain/current-visitors", StatsController, :current_visitors get "/:domain/main-graph", StatsController, :main_graph get "/:domain/top-stats", StatsController, :top_stats diff --git a/lib/plausible_web/templates/layout/app.html.eex b/lib/plausible_web/templates/layout/app.html.eex index a9d79c8ee..b58a7a295 100644 --- a/lib/plausible_web/templates/layout/app.html.eex +++ b/lib/plausible_web/templates/layout/app.html.eex @@ -8,6 +8,7 @@ <%= if Plausible.Funnels.enabled_for?(@conn.assigns[:current_user]) do %> + <% end %> "> diff --git a/lib/plausible_web/templates/layout/focus.html.eex b/lib/plausible_web/templates/layout/focus.html.eex index 3cfa049fa..64f20ad5e 100644 --- a/lib/plausible_web/templates/layout/focus.html.eex +++ b/lib/plausible_web/templates/layout/focus.html.eex @@ -5,6 +5,7 @@ + "> <%= assigns[:title] || "Plausible · Web analytics" %> "/> diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exs index c7ce0800e..2a2e5cd48 100644 --- a/test/plausible_web/controllers/api/internal_controller_test.exs +++ b/test/plausible_web/controllers/api/internal_controller_test.exs @@ -20,6 +20,21 @@ defmodule PlausibleWeb.Api.InternalControllerTest do assert json_response(conn, 200) == "READY" end + + test "is WAITING when unauthenticated", %{user: user} do + site = insert(:site, members: [user]) + Plausible.TestUtils.create_pageviews([%{site: site}]) + + conn = get(build_conn(), "/api/#{site.domain}/status") + + assert json_response(conn, 200) == "WAITING" + end + + test "is WAITING when non-existing site", %{conn: conn} do + conn = get(conn, "/api/example.com/status") + + assert json_response(conn, 200) == "WAITING" + end end describe "GET /api/sites" do diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 7529d6367..048e2d631 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -1,6 +1,7 @@ defmodule PlausibleWeb.StatsControllerTest do use PlausibleWeb.ConnCase, async: true use Plausible.Repo + import Plausible.Test.Support.HTML describe "GET /:website - anonymous user" do test "public site - shows site stats", %{conn: conn} do @@ -8,7 +9,31 @@ defmodule PlausibleWeb.StatsControllerTest do populate_stats(site, [build(:pageview)]) conn = get(conn, "/#{site.domain}") - assert html_response(conn, 200) =~ "stats-react-container" + resp = html_response(conn, 200) + assert element_exists?(resp, "div#stats-react-container") + + assert ["noindex, nofollow"] == + resp + |> find("meta[name=robots]") + |> Floki.attribute("content") + + assert text_of_element(resp, "title") == "Plausible · #{site.domain}" + end + + test "plausible.io live demo - shows site stats", %{conn: conn} do + site = insert(:site, domain: "plausible.io", public: true) + populate_stats(site, [build(:pageview)]) + + conn = get(conn, "/#{site.domain}") + resp = html_response(conn, 200) + assert element_exists?(resp, "div#stats-react-container") + + assert ["index, nofollow"] == + resp + |> find("meta[name=robots]") + |> Floki.attribute("content") + + assert text_of_element(resp, "title") == "Plausible Analytics: Live Demo" end test "public site - shows waiting for first pageview", %{conn: conn} do diff --git a/test/plausible_web/plugs/no_robots_test.exs b/test/plausible_web/plugs/no_robots_test.exs new file mode 100644 index 000000000..1897b4c87 --- /dev/null +++ b/test/plausible_web/plugs/no_robots_test.exs @@ -0,0 +1,36 @@ +defmodule PlausibleWeb.Plugs.NoRobotsTest do + use Plausible.DataCase, async: true + use Plug.Test + + alias PlausibleWeb.Plugs.NoRobots + + test "non-bots pass - when no user agent is supplied" do + conn = :get |> conn("/") |> NoRobots.call() + assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"] + assert get_resp_header(conn, "x-plausible-forbidden-reason") == [] + assert conn.private.robots == "noindex, nofollow" + + refute conn.halted + refute conn.status + end + + test "non-bots pass - when user agent is supplied" do + conn = :get |> conn("/") |> NoRobots.call() + assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"] + assert get_resp_header(conn, "x-plausible-forbidden-reason") == [] + assert conn.private.robots == "noindex, nofollow" + + refute conn.halted + refute conn.status + end + + test "writes index, nofollow for plausible.io live demo" do + conn = :get |> conn("/plausible.io") |> NoRobots.call() + + assert get_resp_header(conn, "x-robots-tag") == ["index, nofollow"] + assert get_resp_header(conn, "x-plausible-forbidden-reason") == [] + + refute conn.halted + refute conn.status + end +end