Fix favicons (#2257)

pull/2261/head
Uku Taht 4 years ago committed by GitHub
parent 81a4e92b0a
commit e849e03058
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -62,4 +62,5 @@ config :geolix,
]
config :plausible,
session_timeout: 0
session_timeout: 0,
http_impl: Plausible.HTTPClient.Mock

@ -1,49 +1,64 @@
defmodule Plausible.HTTPClient do
@moduledoc """
HTTP Client built on top of Finch.
By default, request parameters are json-encoded.
If a raw binary value is supplied, no encoding is performed.
If x-www-form-urlencoded content-type is set in headers,
URL encoding is invoked.
"""
defmodule Plausible.HTTPClient.Non200Error do
defstruct reason: nil
defmodule Non200Error do
defstruct reason: nil
@type t :: %__MODULE__{reason: Finch.Response.t()}
@type t :: %__MODULE__{reason: Finch.Response.t()}
@spec new(Finch.Response.t()) :: t()
def new(%Finch.Response{status: status} = response)
when is_integer(status) and (status < 200 or status >= 300) do
%__MODULE__{reason: response}
end
@spec new(Finch.Response.t()) :: t()
def new(%Finch.Response{status: status} = response)
when is_integer(status) and (status < 200 or status >= 300) do
%__MODULE__{reason: response}
end
end
defmodule Plausible.HTTPClient.Interface do
@type url() :: Finch.Request.url()
@type headers() :: Finch.Request.headers()
@type params() :: Finch.Request.body() | map()
@type response() ::
{:ok, Finch.Response.t()}
| {:error, Mint.Types.error() | Finch.Error.t() | Non200Error.t()}
| {:error, Mint.Types.error() | Finch.Error.t() | Plausible.HTTPClient.Non200Error.t()}
@callback get(url(), headers()) :: response()
@callback get(url()) :: response()
@callback post(url(), headers(), params()) :: response()
end
defmodule Plausible.HTTPClient do
@moduledoc """
HTTP Client built on top of Finch.
By default, request parameters are json-encoded.
If a raw binary value is supplied, no encoding is performed.
If x-www-form-urlencoded content-type is set in headers,
URL encoding is invoked.
"""
@doc """
Make a POST request
"""
@spec post(url(), headers(), params()) :: response()
def(post(url, headers \\ [], params \\ nil)) do
@behaviour Plausible.HTTPClient.Interface
@impl Plausible.HTTPClient.Interface
def post(url, headers \\ [], params \\ nil) do
call(:post, url, headers, params)
end
@doc """
Make a GET request
"""
@spec get(url(), headers()) :: response()
@impl Plausible.HTTPClient.Interface
def get(url, headers \\ []) do
call(:get, url, headers, nil)
end
# TODO: Is it possible to tell the type checker that we're returning a module that conforms to the
# Plausible.HTTPClient.Interface behaviour?
@spec impl() :: Plausible.HTTPClient
def impl() do
Application.get_env(:plausible, :http_impl, __MODULE__)
end
defp call(method, url, headers, params) do
{params, headers} = maybe_encode_params(params, headers)
@ -91,7 +106,7 @@ defmodule Plausible.HTTPClient do
end
defp tag_error({:ok, %Finch.Response{status: _} = response}) do
{:error, Non200Error.new(response)}
{:error, Plausible.HTTPClient.Non200Error.new(response)}
end
defp tag_error({:error, _} = error) do

@ -1,29 +1,58 @@
defmodule PlausibleWeb.Favicon do
@referer_domains_file "priv/referer_favicon_domains.json"
@moduledoc """
A Plug that fetches favicon images from DuckDuckGo and returns them
to the Plausible frontend.
The proxying is there so we can reduce the number of third-party domains that
the browser clients need to connect to. Our goal is to have 0 third-party domain
connections on the website for privacy reasons.
This module also maps between categorized sources and their respective URLs for favicons.
What does that mean exactly? During ingestion we use `PlausibleWeb.RefInspector.parse/1` to
categorize our referrer sources like so:
google.com -> Google
google.co.uk -> Google
google.com.au -> Google
So when we show Google as a source in the dashboard, the request to this plug will come as:
https://plausible/io/favicon/sources/Google
Now, when we want to show a favicon for Google, we need to convert Google -> google.com or
some other hostname owned by Google:
https://icons.duckduckgo.com/ip3/google.com.ico
The mapping from source category -> source hostname is stored in "#{@referer_domains_file}" and
managed by `Mix.Tasks.GenerateReferrerFavicons.run/1`
"""
import Plug.Conn
alias Plausible.HTTPClient
def init(_) do
domains =
case File.read(Application.app_dir(:plausible, "priv/referer_favicon_domains.json")) do
{:ok, contents} ->
Jason.decode!(contents)
_ ->
%{}
end
File.read!(Application.app_dir(:plausible, @referer_domains_file))
|> Jason.decode!()
[favicon_domains: domains]
end
@doc """
Proxies HTTP request to DuckDuckGo favicon service. Swallows hop-by-hop HTTP headers that
should not be forwarded as defined in RFC 2616 (https://www.rfc-editor.org/rfc/rfc2616#section-13.5.1)
"""
def call(conn, favicon_domains: favicon_domains) do
case conn.path_info do
["favicon", "sources", source] ->
clean_source = URI.decode_www_form(source)
domain = Map.get(favicon_domains, clean_source, clean_source)
case HTTPClient.get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do
case HTTPClient.impl().get("https://icons.duckduckgo.com/ip3/#{domain}.ico") do
{:ok, res} ->
send_response(conn, res)
conn
|> forward_headers(res)
|> send_resp(200, res.body)
|> halt
_ ->
send_resp(conn, 503, "") |> halt
@ -34,13 +63,9 @@ defmodule PlausibleWeb.Favicon do
end
end
defp send_response(conn, res) do
conn =
Enum.filter(res.headers, fn {key, _val} -> key != "Transfer-Encoding" end)
|> Enum.reduce(conn, fn {key, val}, conn ->
put_resp_header(conn, key, val)
end)
send_resp(conn, 200, res.body) |> halt
@forwarded_headers ["content-type", "cache-control", "expires"]
defp forward_headers(conn, response) do
headers = Enum.filter(response.headers, fn {k, _} -> k in @forwarded_headers end)
%Plug.Conn{conn | resp_headers: headers}
end
end

@ -88,6 +88,7 @@ defmodule Plausible.MixProject do
{:kaffy, "~> 0.9.0"},
{:location, git: "https://github.com/plausible/location.git"},
{:mimic, "~> 1.7", only: :test},
{:mox, "~> 1.0", only: :test},
{:nanoid, "~> 2.0.2"},
{:oauther, "~> 1.3"},
{:oban, "~> 2.11"},

@ -76,6 +76,7 @@
"mimic": {:hex, :mimic, "1.7.2", "27007e4e0c746ddb6d56a386c40585088b35621ae2d7167160e8c3283e8cd585", [:mix], [], "hexpm", "e4d40550523841055aa469f5125d124ab89ce8b2d3686cab908b98dff5e6111b"},
"mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"},
"mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.1", "78e3aedde88035c6873ada5ceaf41b7f15a6259ed034e0eaca72ccfa937798f0", [:mix], [], "hexpm", "316af0f388fac824782d944f54efe78e7c9691bbbdb0afd5cccdd0510adf559d"},
"mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"},
"nanoid": {:hex, :nanoid, "2.0.5", "1d2948d8967ef2d948a58c3fef02385040bd9823fc6394bd604b8d98e5516b22", [:mix], [], "hexpm", "956e8876321104da72aa48770539ff26b36b744cd26753ec8e7a8a37e53d5f58"},
"nimble_options": {:hex, :nimble_options, "0.4.0", "c89babbab52221a24b8d1ff9e7d838be70f0d871be823165c94dd3418eea728f", [:mix], [], "hexpm", "e6701c1af326a11eea9634a3b1c62b475339ace9456c1a23ec3bc9a847bca02d"},
"nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},

@ -122,6 +122,16 @@ defmodule Plausible.HTTPClientTest do
}} = HTTPClient.get(bypass_url(bypass, path: "/get"))
end
test "header keys are downcased but values are not", %{bypass: bypass} do
Bypass.expect_once(bypass, "GET", "/get", fn conn ->
Conn.resp(conn, 200, "ok")
|> Conn.put_resp_header("Some-Header", "Header-Value")
end)
assert {:ok, res} = HTTPClient.get(bypass_url(bypass, path: "/get"))
assert {"some-header", "Header-Value"} in res.headers
end
defp bypass_url(bypass, opts) do
port = bypass.port
path = Keyword.get(opts, :path, "/")

@ -0,0 +1,87 @@
defmodule PlausibleWeb.FaviconTest do
use Plausible.DataCase
use Plug.Test
alias PlausibleWeb.Favicon
import Mox
setup :verify_on_exit!
setup_all do
opts = PlausibleWeb.Favicon.init(nil)
%{plug_opts: opts}
end
test "ignores request on a URL it does not need to handle", %{plug_opts: plug_opts} do
old_conn = conn(:get, "/irrelevant")
new_conn = Favicon.call(old_conn, plug_opts)
refute new_conn.halted
assert old_conn == new_conn
end
test "proxies request on favicon URL to duckduckgo", %{plug_opts: plug_opts} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->
{:ok, %Finch.Response{status: 200, body: "favicon response body"}}
end
)
conn =
conn(:get, "/favicon/sources/plausible.io")
|> Favicon.call(plug_opts)
assert conn.halted
assert conn.status == 200
assert conn.resp_body == "favicon response body"
end
test "maps a categorized source to URL for favicon", %{plug_opts: plug_opts} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn "https://icons.duckduckgo.com/ip3/facebook.com.ico" ->
{:ok, %Finch.Response{status: 200, body: "favicon response body"}}
end
)
conn =
conn(:get, "/favicon/sources/Facebook")
|> Favicon.call(plug_opts)
assert conn.halted
assert conn.status == 200
assert conn.resp_body == "favicon response body"
end
test "copies content-type header from the proxied response", %{plug_opts: plug_opts} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn "https://icons.duckduckgo.com/ip3/plausible.io.ico" ->
{:ok,
%Finch.Response{
status: 200,
body: "favicon response body",
headers: [
{"transfer-encoding", "chunked"},
{"content-type", "should-pass-through"}
]
}}
end
)
conn =
conn(:get, "/favicon/sources/plausible.io")
|> Favicon.call(plug_opts)
assert conn.halted
assert conn.status == 200
assert conn.resp_headers == [
{"content-type", "should-pass-through"}
]
end
end

@ -1,6 +1,7 @@
{:ok, _} = Application.ensure_all_started(:ex_machina)
Plausible.Test.ClickhouseSetup.run()
Mimic.copy(FunWithFlags)
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
ExUnit.start()
Application.ensure_all_started(:double)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)

Loading…
Cancel
Save