From e849e030586d410bde613ded4c820df2bb242b80 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 23 Sep 2022 13:22:43 +0300 Subject: [PATCH] Fix favicons (#2257) --- config/test.exs | 3 +- lib/plausible/http_client.ex | 63 +++++++++------- lib/plausible_web/plugs/favicon.ex | 59 ++++++++++----- mix.exs | 1 + mix.lock | 1 + test/plausible/http_client_test.exs | 10 +++ test/plausible_web/plugs/favicon_test.exs | 87 +++++++++++++++++++++++ test/test_helper.exs | 1 + 8 files changed, 183 insertions(+), 42 deletions(-) create mode 100644 test/plausible_web/plugs/favicon_test.exs diff --git a/config/test.exs b/config/test.exs index c311c15a9..d148d4ca1 100644 --- a/config/test.exs +++ b/config/test.exs @@ -62,4 +62,5 @@ config :geolix, ] config :plausible, - session_timeout: 0 + session_timeout: 0, + http_impl: Plausible.HTTPClient.Mock diff --git a/lib/plausible/http_client.ex b/lib/plausible/http_client.ex index c65f5331d..5de47e095 100644 --- a/lib/plausible/http_client.ex +++ b/lib/plausible/http_client.ex @@ -1,49 +1,64 @@ -defmodule Plausible.HTTPClient do - @moduledoc """ - HTTP Client built on top of Finch. - - By default, request parameters are json-encoded. - - If a raw binary value is supplied, no encoding is performed. - If x-www-form-urlencoded content-type is set in headers, - URL encoding is invoked. - """ +defmodule Plausible.HTTPClient.Non200Error do + defstruct reason: nil - defmodule Non200Error do - defstruct reason: nil + @type t :: %__MODULE__{reason: Finch.Response.t()} - @type t :: %__MODULE__{reason: Finch.Response.t()} - - @spec new(Finch.Response.t()) :: t() - def new(%Finch.Response{status: status} = response) - when is_integer(status) and (status < 200 or status >= 300) do - %__MODULE__{reason: response} - end + @spec new(Finch.Response.t()) :: t() + def new(%Finch.Response{status: status} = response) + when is_integer(status) and (status < 200 or status >= 300) do + %__MODULE__{reason: response} end +end +defmodule Plausible.HTTPClient.Interface do @type url() :: Finch.Request.url() @type headers() :: Finch.Request.headers() @type params() :: Finch.Request.body() | map() @type response() :: {:ok, Finch.Response.t()} - | {:error, Mint.Types.error() | Finch.Error.t() | Non200Error.t()} + | {:error, Mint.Types.error() | Finch.Error.t() | Plausible.HTTPClient.Non200Error.t()} + + @callback get(url(), headers()) :: response() + @callback get(url()) :: response() + @callback post(url(), headers(), params()) :: response() +end + +defmodule Plausible.HTTPClient do + @moduledoc """ + HTTP Client built on top of Finch. + + By default, request parameters are json-encoded. + + If a raw binary value is supplied, no encoding is performed. + If x-www-form-urlencoded content-type is set in headers, + URL encoding is invoked. + """ @doc """ Make a POST request """ - @spec post(url(), headers(), params()) :: response() - def(post(url, headers \\ [], params \\ nil)) do + @behaviour Plausible.HTTPClient.Interface + + @impl Plausible.HTTPClient.Interface + def post(url, headers \\ [], params \\ nil) do call(:post, url, headers, params) end @doc """ Make a GET request """ - @spec get(url(), headers()) :: response() + @impl Plausible.HTTPClient.Interface def get(url, headers \\ []) do call(:get, url, headers, nil) end + # TODO: Is it possible to tell the type checker that we're returning a module that conforms to the + # Plausible.HTTPClient.Interface behaviour? + @spec impl() :: Plausible.HTTPClient + def impl() do + Application.get_env(:plausible, :http_impl, __MODULE__) + end + defp call(method, url, headers, params) do {params, headers} = maybe_encode_params(params, headers) @@ -91,7 +106,7 @@ defmodule Plausible.HTTPClient do end defp tag_error({:ok, %Finch.Response{status: _} = response}) do - {:error, Non200Error.new(response)} + {:error, Plausible.HTTPClient.Non200Error.new(response)} end defp tag_error({:error, _} = error) do diff --git a/lib/plausible_web/plugs/favicon.ex b/lib/plausible_web/plugs/favicon.ex index d60e4d326..02bce94b0 100644 --- a/lib/plausible_web/plugs/favicon.ex +++ b/lib/plausible_web/plugs/favicon.ex @@ -1,29 +1,58 @@ defmodule PlausibleWeb.Favicon do + @referer_domains_file "priv/referer_favicon_domains.json" + @moduledoc """ + A Plug that fetches favicon images from DuckDuckGo and returns them + to the Plausible frontend. + + The proxying is there so we can reduce the number of third-party domains that + the browser clients need to connect to. Our goal is to have 0 third-party domain + connections on the website for privacy reasons. + + This module also maps between categorized sources and their respective URLs for favicons. + What does that mean exactly? During ingestion we use `PlausibleWeb.RefInspector.parse/1` to + categorize our referrer sources like so: + + google.com -> Google + google.co.uk -> Google + google.com.au -> Google + + So when we show Google as a source in the dashboard, the request to this plug will come as: + https://plausible/io/favicon/sources/Google + + Now, when we want to show a favicon for Google, we need to convert Google -> google.com or + some other hostname owned by Google: + https://icons.duckduckgo.com/ip3/google.com.ico + + The mapping from source category -> source hostname is stored in "#{@referer_domains_file}" and + managed by `Mix.Tasks.GenerateReferrerFavicons.run/1` + """ import Plug.Conn alias Plausible.HTTPClient def init(_) do domains = - case File.read(Application.app_dir(:plausible, "priv/referer_favicon_domains.json")) do - {:ok, contents} -> - Jason.decode!(contents) - - _ -> - %{} - end + File.read!(Application.app_dir(:plausible, @referer_domains_file)) + |> Jason.decode!() [favicon_domains: domains] end + @doc """ + Proxies HTTP request to DuckDuckGo favicon service. Swallows hop-by-hop HTTP headers that + should not be forwarded as defined in RFC 2616 (https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1) + """ def call(conn, favicon_domains: favicon_domains) do case conn.path_info do ["favicon", "sources", source] -> clean_source = URI.decode_www_form(source) domain = Map.get(favicon_domains, clean_source, clean_source) - case HTTPClient.get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do + case HTTPClient.impl().get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do {:ok, res} -> - send_response(conn, res) + conn + |> forward_headers(res) + |> send_resp(200, res.body) + |> halt _ -> send_resp(conn, 503, "") |> halt @@ -34,13 +63,9 @@ defmodule PlausibleWeb.Favicon do end end - defp send_response(conn, res) do - conn = - Enum.filter(res.headers, fn {key, _val} -> key != "Transfer-Encoding" end) - |> Enum.reduce(conn, fn {key, val}, conn -> - put_resp_header(conn, key, val) - end) - - send_resp(conn, 200, res.body) |> halt + @forwarded_headers ["content-type", "cache-control", "expires"] + defp forward_headers(conn, response) do + headers = Enum.filter(response.headers, fn {k, _} -> k in @forwarded_headers end) + %Plug.Conn{conn | resp_headers: headers} end end diff --git a/mix.exs b/mix.exs index e3b8cd8bc..b56e72648 100644 --- a/mix.exs +++ b/mix.exs @@ -88,6 +88,7 @@ defmodule Plausible.MixProject do {:kaffy, "~> 0.9.0"}, {:location, git: "https://github.com/plausible/location.git"}, {:mimic, "~> 1.7", only: :test}, + {:mox, "~> 1.0", only: :test}, {:nanoid, "~> 2.0.2"}, {:oauther, "~> 1.3"}, {:oban, "~> 2.11"}, diff --git a/mix.lock b/mix.lock index c24149f13..c8fd7411e 100644 --- a/mix.lock +++ b/mix.lock @@ -76,6 +76,7 @@ "mimic": {:hex, :mimic, "1.7.2", "27007e4e0c746ddb6d56a386c40585088b35621ae2d7167160e8c3283e8cd585", [:mix], [], "hexpm", "e4d40550523841055aa469f5125d124ab89ce8b2d3686cab908b98dff5e6111b"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, "mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.1", "78e3aedde88035c6873ada5ceaf41b7f15a6259ed034e0eaca72ccfa937798f0", [:mix], [], "hexpm", "316af0f388fac824782d944f54efe78e7c9691bbbdb0afd5cccdd0510adf559d"}, + "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, "nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"}, "nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"}, "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, diff --git a/test/plausible/http_client_test.exs b/test/plausible/http_client_test.exs index b38c3dabc..669270d4f 100644 --- a/test/plausible/http_client_test.exs +++ b/test/plausible/http_client_test.exs @@ -122,6 +122,16 @@ defmodule Plausible.HTTPClientTest do }} = HTTPClient.get(bypass_url(bypass, path: "/get")) end + test "header keys are downcased but values are not", %{bypass: bypass} do + Bypass.expect_once(bypass, "GET", "/get", fn conn -> + Conn.resp(conn, 200, "ok") + |> Conn.put_resp_header("Some-Header", "Header-Value") + end) + + assert {:ok, res} = HTTPClient.get(bypass_url(bypass, path: "/get")) + assert {"some-header", "Header-Value"} in res.headers + end + defp bypass_url(bypass, opts) do port = bypass.port path = Keyword.get(opts, :path, "/") diff --git a/test/plausible_web/plugs/favicon_test.exs b/test/plausible_web/plugs/favicon_test.exs new file mode 100644 index 000000000..b15beca00 --- /dev/null +++ b/test/plausible_web/plugs/favicon_test.exs @@ -0,0 +1,87 @@ +defmodule PlausibleWeb.FaviconTest do + use Plausible.DataCase + use Plug.Test + alias PlausibleWeb.Favicon + + import Mox + setup :verify_on_exit! + + setup_all do + opts = PlausibleWeb.Favicon.init(nil) + + %{plug_opts: opts} + end + + test "ignores request on a URL it does not need to handle", %{plug_opts: plug_opts} do + old_conn = conn(:get, "/irrelevant") + new_conn = Favicon.call(old_conn, plug_opts) + + refute new_conn.halted + assert old_conn == new_conn + end + + test "proxies request on favicon URL to duckduckgo", %{plug_opts: plug_opts} do + expect( + Plausible.HTTPClient.Mock, + :get, + fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" -> + {:ok, %Finch.Response{status: 200, body: "favicon response body"}} + end + ) + + conn = + conn(:get, "/favicon/sources/plausible.io") + |> Favicon.call(plug_opts) + + assert conn.halted + assert conn.status == 200 + assert conn.resp_body == "favicon response body" + end + + test "maps a categorized source to URL for favicon", %{plug_opts: plug_opts} do + expect( + Plausible.HTTPClient.Mock, + :get, + fn "https://icons.duckduckgo.com/ip3/facebook.com.ico" -> + {:ok, %Finch.Response{status: 200, body: "favicon response body"}} + end + ) + + conn = + conn(:get, "/favicon/sources/Facebook") + |> Favicon.call(plug_opts) + + assert conn.halted + assert conn.status == 200 + assert conn.resp_body == "favicon response body" + end + + test "copies content-type header from the proxied response", %{plug_opts: plug_opts} do + expect( + Plausible.HTTPClient.Mock, + :get, + fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" -> + {:ok, + %Finch.Response{ + status: 200, + body: "favicon response body", + headers: [ + {"transfer-encoding", "chunked"}, + {"content-type", "should-pass-through"} + ] + }} + end + ) + + conn = + conn(:get, "/favicon/sources/plausible.io") + |> Favicon.call(plug_opts) + + assert conn.halted + assert conn.status == 200 + + assert conn.resp_headers == [ + {"content-type", "should-pass-through"} + ] + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 78603315c..3fb7b96fe 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,6 +1,7 @@ {:ok, _} = Application.ensure_all_started(:ex_machina) Plausible.Test.ClickhouseSetup.run() Mimic.copy(FunWithFlags) +Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface) ExUnit.start() Application.ensure_all_started(:double) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)