Prevent robots from looking too much into public dashboards (#3118)
* Add noindex,nofollow to dashboard pages * Implement NoRobots plug * Enable NoRobots plug in the router * Fixup internal route * Fix double slash in the router * Add special bot treatment to plausible.io live demo page * Revert aggressive protection with agent detectionpull/3021/head
parent
fd3c9ecc7b
commit
fea9bb32ee
@ -0,0 +1,29 @@
|
||||
defmodule PlausibleWeb.Plugs.NoRobots do
|
||||
@moduledoc """
|
||||
Rejects bot requests by any means available.
|
||||
|
||||
We're adding `x-robots-tag` to the response header and annotate the conn
|
||||
with "noindex, nofollow" under `private.robots` key.
|
||||
|
||||
The only exception is, if the request is trying to access our live demo
|
||||
at plausible.io/plausible.io - in which case we'll allow indexing, but deny
|
||||
following links and skip the bot detection, in kind robots we trust.
|
||||
"""
|
||||
@behaviour Plug
|
||||
import Plug.Conn
|
||||
|
||||
@impl true
|
||||
def init(opts), do: opts
|
||||
|
||||
@impl true
|
||||
def call(conn, _opts \\ nil) do
|
||||
conn =
|
||||
if conn.path_info == ["plausible.io"] do
|
||||
put_private(conn, :robots, "index, nofollow")
|
||||
else
|
||||
put_private(conn, :robots, "noindex, nofollow")
|
||||
end
|
||||
|
||||
put_resp_header(conn, "x-robots-tag", conn.private.robots)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,36 @@
|
||||
defmodule PlausibleWeb.Plugs.NoRobotsTest do
|
||||
use Plausible.DataCase, async: true
|
||||
use Plug.Test
|
||||
|
||||
alias PlausibleWeb.Plugs.NoRobots
|
||||
|
||||
test "non-bots pass - when no user agent is supplied" do
|
||||
conn = :get |> conn("/") |> NoRobots.call()
|
||||
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
|
||||
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
|
||||
assert conn.private.robots == "noindex, nofollow"
|
||||
|
||||
refute conn.halted
|
||||
refute conn.status
|
||||
end
|
||||
|
||||
test "non-bots pass - when user agent is supplied" do
|
||||
conn = :get |> conn("/") |> NoRobots.call()
|
||||
assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
|
||||
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
|
||||
assert conn.private.robots == "noindex, nofollow"
|
||||
|
||||
refute conn.halted
|
||||
refute conn.status
|
||||
end
|
||||
|
||||
test "writes index, nofollow for plausible.io live demo" do
|
||||
conn = :get |> conn("/plausible.io") |> NoRobots.call()
|
||||
|
||||
assert get_resp_header(conn, "x-robots-tag") == ["index, nofollow"]
|
||||
assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
|
||||
|
||||
refute conn.halted
|
||||
refute conn.status
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue