From 64f35bf8f6188625bacade2f8f2e7687f2b73e22 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Tue, 19 Mar 2024 19:07:30 +0800 Subject: [PATCH] make robots.txt restrictive by default (#3905) * make robots.txt restrictive by default * add note about cloud robots.txt --- CHANGELOG.md | 1 + lib/plausible_web/endpoint.ex | 12 +++++++++++- priv/static/robots.txt | 7 ++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e0c99d44..43f817873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project will be documented in this file. - Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB) - Add alternative SMTP adapter plausible/analytics#3654 - Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906 +- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905 ### Removed - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions diff --git a/lib/plausible_web/endpoint.ex b/lib/plausible_web/endpoint.ex index 5b6c6f930..0cc4c91b9 100644 --- a/lib/plausible_web/endpoint.ex +++ b/lib/plausible_web/endpoint.ex @@ -33,11 +33,21 @@ defmodule PlausibleWeb.Endpoint do plug(PlausibleWeb.Tracker) plug(PlausibleWeb.Favicon) + static_paths = ~w(css js images favicon.ico) + + static_paths = + on_full_build do + # NOTE: The Cloud uses custom robots.txt from https://github.com/plausible/website: https://plausible.io/robots.txt + static_paths + else + static_paths ++ ["robots.txt"] + end + plug(Plug.Static, at: "/", from: :plausible, gzip: false, - only: ~w(css js images favicon.ico robots.txt) + only: static_paths ) on_full_build do diff --git a/priv/static/robots.txt b/priv/static/robots.txt index 3c9c7c01f..1f53798bb 100644 --- a/priv/static/robots.txt +++ b/priv/static/robots.txt @@ -1,5 +1,2 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +User-agent: * +Disallow: /