Prevent robots from looking too much into public dashboards (#3118)

* Add noindex,nofollow to dashboard pages

* Implement NoRobots plug

* Enable NoRobots plug in the router

* Fixup internal route

* Fix double slash in the router

* Add special bot treatment to plausible.io live demo page

* Revert aggressive protection with agent detection
pull/3021/head
hq1 3 years ago committed by GitHub
parent fd3c9ecc7b
commit fea9bb32ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -6,12 +6,14 @@ defmodule PlausibleWeb.Api.InternalController do
alias Plausible.Auth.User
def domain_status(conn, %{"domain" => domain}) do
site = Sites.get_by_domain(domain)
if Stats.has_pageviews?(site) do
with %User{id: user_id} <- conn.assigns[:current_user],
%Site{} = site <- Sites.get_by_domain(domain),
true <- Sites.has_admin_access?(user_id, site) || Auth.is_super_admin?(user_id),
true <- Stats.has_pageviews?(site) do
json(conn, "READY")
else
json(conn, "WAITING")
_ ->
json(conn, "WAITING")
end
end

@ -67,7 +67,7 @@ defmodule PlausibleWeb.StatsController do
funnels: Plausible.Funnels.list(site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
title: "Plausible · " <> site.domain,
title: title(conn, site),
offer_email_report: offer_email_report,
demo: demo,
flags: get_flags(conn.assigns[:current_user]),
@ -295,7 +295,7 @@ defmodule PlausibleWeb.StatsController do
funnels: Plausible.Funnels.list(shared_link.site),
stats_start_date: shared_link.site.stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: "Plausible · " <> shared_link.site.domain,
title: title(conn, shared_link.site),
offer_email_report: false,
demo: false,
skip_plausible_tracking: true,
@ -343,4 +343,12 @@ defmodule PlausibleWeb.StatsController do
!!is_or_nil
end
defp title(%{path_info: ["plausible.io"]}, _) do
"Plausible Analytics: Live Demo"
end
defp title(_conn, site) do
"Plausible · " <> site.domain
end
end

@ -0,0 +1,29 @@
defmodule PlausibleWeb.Plugs.NoRobots do
@moduledoc """
Rejects bot requests by any means available.
We're adding `x-robots-tag` to the response header and annotate the conn
with "noindex, nofollow" under `private.robots` key.
The only exception is, if the request is trying to access our live demo
at plausible.io/plausible.io - in which case we'll allow indexing, but deny
following links and skip the bot detection, in kind robots we trust.
"""
@behaviour Plug
import Plug.Conn
@impl true
def init(opts), do: opts
@impl true
def call(conn, _opts \\ nil) do
conn =
if conn.path_info == ["plausible.io"] do
put_private(conn, :robots, "index, nofollow")
else
put_private(conn, :robots, "noindex, nofollow")
end
put_resp_header(conn, "x-robots-tag", conn.private.robots)
end
end

@ -8,6 +8,7 @@ defmodule PlausibleWeb.Router do
plug :fetch_session
plug :fetch_live_flash
plug :put_secure_browser_headers
plug PlausibleWeb.Plugs.NoRobots
plug PlausibleWeb.FirstLaunchPlug, redirect_to: "/register"
plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_seconds
plug PlausibleWeb.AuthPlug
@ -17,6 +18,7 @@ defmodule PlausibleWeb.Router do
pipeline :shared_link do
plug :accepts, ["html"]
plug :put_secure_browser_headers
plug PlausibleWeb.Plugs.NoRobots
end
pipeline :csrf do
@ -33,6 +35,7 @@ defmodule PlausibleWeb.Router do
plug :accepts, ["json"]
plug :fetch_session
plug PlausibleWeb.AuthorizeSiteAccess
plug PlausibleWeb.Plugs.NoRobots
end
pipeline :public_api do
@ -42,6 +45,7 @@ defmodule PlausibleWeb.Router do
pipeline :flags do
plug :accepts, ["html"]
plug :put_secure_browser_headers
plug PlausibleWeb.Plugs.NoRobots
plug :fetch_session
plug PlausibleWeb.CRMAuthPlug
end
@ -50,7 +54,9 @@ defmodule PlausibleWeb.Router do
forward "/sent-emails", Bamboo.SentEmailViewerPlug
end
use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug]
use Kaffy.Routes,
scope: "/crm",
pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug]
scope path: "/flags" do
pipe_through :flags
@ -59,7 +65,7 @@ defmodule PlausibleWeb.Router do
scope "/api/stats", PlausibleWeb.Api do
pipe_through :internal_stats_api
get "/:domain//funnels/:id", StatsController, :funnel
get "/:domain/funnels/:id", StatsController, :funnel
get "/:domain/current-visitors", StatsController, :current_visitors
get "/:domain/main-graph", StatsController, :main_graph
get "/:domain/top-stats", StatsController, :top_stats

@ -8,6 +8,7 @@
<%= if Plausible.Funnels.enabled_for?(@conn.assigns[:current_user]) do %>
<meta name="csrf-token" content="<%= Plug.CSRFProtection.get_csrf_token() %>" />
<meta name="websocket-url" content="<%= websocket_url() %>" />
<meta name="robots" content="<%= @conn.private.robots %>" />
<% end %>
<link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>">
<link rel="apple-touch-icon" href="/images/icon/apple-touch-icon.png">

@ -5,6 +5,7 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<meta name="description" content="A lightweight, non-intrusive alternative to Google Analytics."/>
<meta name="robots" content="<%= @conn.private.robots %>" />
<link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>">
<title><%= assigns[:title] || "Plausible · Web analytics" %></title>
<link rel="stylesheet" href="<%= Routes.static_path(@conn, "/css/app.css") %>"/>

@ -20,6 +20,21 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
assert json_response(conn, 200) == "READY"
end
test "is WAITING when unauthenticated", %{user: user} do
site = insert(:site, members: [user])
Plausible.TestUtils.create_pageviews([%{site: site}])
conn = get(build_conn(), "/api/#{site.domain}/status")
assert json_response(conn, 200) == "WAITING"
end
test "is WAITING when non-existing site", %{conn: conn} do
conn = get(conn, "/api/example.com/status")
assert json_response(conn, 200) == "WAITING"
end
end
describe "GET /api/sites" do

@ -1,6 +1,7 @@
defmodule PlausibleWeb.StatsControllerTest do
use PlausibleWeb.ConnCase, async: true
use Plausible.Repo
import Plausible.Test.Support.HTML
describe "GET /:website - anonymous user" do
test "public site - shows site stats", %{conn: conn} do
@ -8,7 +9,31 @@ defmodule PlausibleWeb.StatsControllerTest do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/#{site.domain}")
assert html_response(conn, 200) =~ "stats-react-container"
resp = html_response(conn, 200)
assert element_exists?(resp, "div#stats-react-container")
assert ["noindex, nofollow"] ==
resp
|> find("meta[name=robots]")
|> Floki.attribute("content")
assert text_of_element(resp, "title") == "Plausible · #{site.domain}"
end
test "plausible.io live demo - shows site stats", %{conn: conn} do
site = insert(:site, domain: "plausible.io", public: true)
populate_stats(site, [build(:pageview)])
conn = get(conn, "/#{site.domain}")
resp = html_response(conn, 200)
assert element_exists?(resp, "div#stats-react-container")
assert ["index, nofollow"] ==
resp
|> find("meta[name=robots]")
|> Floki.attribute("content")
assert text_of_element(resp, "title") == "Plausible Analytics: Live Demo"
end
test "public site - shows waiting for first pageview", %{conn: conn} do

@ -0,0 +1,36 @@
defmodule PlausibleWeb.Plugs.NoRobotsTest do
use Plausible.DataCase, async: true
use Plug.Test
alias PlausibleWeb.Plugs.NoRobots
test "non-bots pass - when no user agent is supplied" do
conn = :get |> conn("/") |> NoRobots.call()
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
assert conn.private.robots == "noindex, nofollow"
refute conn.halted
refute conn.status
end
test "non-bots pass - when user agent is supplied" do
conn = :get |> conn("/") |> NoRobots.call()
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
assert conn.private.robots == "noindex, nofollow"
refute conn.halted
refute conn.status
end
test "writes index, nofollow for plausible.io live demo" do
conn = :get |> conn("/plausible.io") |> NoRobots.call()
assert get_resp_header(conn, "x-robots-tag") == ["index, nofollow"]
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
refute conn.halted
refute conn.status
end
end
Loading…
Cancel
Save