From 3fe6a956960dca2eb53395b0063d2f11066b6bb0 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Wed, 16 Nov 2022 15:04:53 +0100 Subject: [PATCH] Update sites cache implementation (#2444) * Allow refreshing a single site cache + clear cache on prefill * Reorganize Site.Cache tests with describe blocks * Tidy up Cache tests * Make sure the cache is cleared on (p)re-fill * Allow process name customization in Cache.Warmer * s/Cache.prefill/Cache.refresh * Unify Cache refresh instrumentation * Apply credo suggestion: change `with` to `case` * Update typespecs to pass dialyzer --- lib/plausible/site/cache.ex | 117 +++++++++++-- lib/plausible/site/cache/warmer.ex | 38 +--- test/plausible/site/cache_test.exs | 271 ++++++++++++++++++++--------- 3 files changed, 296 insertions(+), 130 deletions(-) diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex index c24a7c0aa..f0df6e33b 100644 --- a/lib/plausible/site/cache.ex +++ b/lib/plausible/site/cache.ex @@ -1,13 +1,13 @@ defmodule Plausible.Site.Cache do @moduledoc """ - A sites by domain caching interface. + A "sites by domain" caching interface. Serves as a thin wrapper around Cachex, but the underlying implementation can be transparently swapped. Even though the Cachex process is started, cache access is disabled during tests via the `:sites_by_domain_cache_enabled` application env key. - This can be overriden on case by case basis, using the child specs options. + This can be overridden on case by case basis, using the child specs options. When Cache is disabled via application env, the `get/1` function falls back to pure database lookups. This should help with introducing @@ -19,8 +19,26 @@ defmodule Plausible.Site.Cache do database counterpart -- to spare bandwidth and query execution time, only selected database columns are retrieved and cached. + There are two modes of refreshing the cache: `:all` and `:single`. + + * `:all` means querying the database for all Site entries and should be done + periodically (via `Cache.Warmer`). All existing Cache entries all cleared + prior to writing the new batch. + + * `:single` attempts to re-query a specific site by domain and should be done + only when the initial Cache.get attempt resulted with `nil`. Single refresh will + write `%Ecto.NoResultsError{}` to the cache so that subsequent Cache.get calls + indicate that we already failed to retrieve a Site. + + This helps in recognising missing/deleted Sites with minimal number of DB lookups + across a disconnected cluster within the periodic refresh window. + + Refreshing a single Site emits a telemetry event including `duration` measurement + and meta-data indicating whether the site was found in the DB or is missing still. + The telemetry event is defined with `telemetry_event_refresh/2`. + The `@cached_schema_fields` attribute defines the list of DB columns - queried on cache pre-fill. + queried on each cache refresh. Also see tests for more comprehensive examples. """ @@ -39,6 +57,9 @@ defmodule Plausible.Site.Cache do ingest_rate_limit_threshold )a + @type not_found_in_db() :: %Ecto.NoResultsError{} + @type t() :: Site.t() | not_found_in_db() + def name(), do: @cache_name @spec child_spec(Keyword.t()) :: Supervisor.child_spec() @@ -52,20 +73,27 @@ defmodule Plausible.Site.Cache do ) end - @spec prefill(Keyword.t()) :: :ok - def prefill(opts) do + @spec refresh_all(Keyword.t()) :: :ok + def refresh_all(opts) do cache_name = Keyword.fetch!(opts, :cache_name) - sites_by_domain_query = - from s in Site, - select: { - s.domain, - %{struct(s, ^@cached_schema_fields) | from_cache?: true} - } + measure_duration(telemetry_event_refresh(cache_name, :all), fn -> + sites_by_domain_query = + from s in Site, + select: { + s.domain, + %{struct(s, ^@cached_schema_fields) | from_cache?: true} + } - sites_by_domain = Plausible.Repo.all(sites_by_domain_query) + sites_by_domain = Plausible.Repo.all(sites_by_domain_query) + + Cachex.clear!(cache_name) + + if not Enum.empty?(sites_by_domain) do + true = Cachex.put_many!(cache_name, sites_by_domain) + end + end) - true = Cachex.put_many!(cache_name, sites_by_domain) :ok end @@ -81,7 +109,7 @@ defmodule Plausible.Site.Cache do Map.get(stats, :hit_rate, 0) end - @spec get(String.t(), Keyword.t()) :: nil | Site.t() + @spec get(String.t(), Keyword.t()) :: t() | nil def get(domain, opts \\ []) do cache_name = Keyword.get(opts, :cache_name, @cache_name) force? = Keyword.get(opts, :force?, false) @@ -106,7 +134,68 @@ defmodule Plausible.Site.Cache do end end + @spec refresh_one(String.t(), Keyword.t()) :: {:ok, t()} | {:error, any()} + def refresh_one(domain, opts) do + cache_name = Keyword.get(opts, :cache_name, @cache_name) + force? = Keyword.get(opts, :force?, false) + + if not enabled?() and not force?, do: raise("Cache: '#{cache_name}' is disabled") + + measure_duration_with_metadata(telemetry_event_refresh(cache_name, :one), fn -> + {found_in_db?, item_to_cache} = select_one(domain) + + case Cachex.put(cache_name, domain, item_to_cache) do + {:ok, _} -> + result = {:ok, item_to_cache} + {result, with_telemetry_metadata(found_in_db?: found_in_db?)} + + {:error, _} = error -> + {error, with_telemetry_metadata(error: true)} + end + end) + end + + @spec telemetry_event_refresh(atom(), :all | :one) :: list(atom()) + def telemetry_event_refresh(cache_name, mode) when mode in [:all, :one] do + [:plausible, :cache, cache_name, :refresh, mode] + end + def enabled?() do Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true end + + defp select_one(domain) do + site_by_domain_query = + from s in Site, + where: s.domain == ^domain, + select: %{struct(s, ^@cached_schema_fields) | from_cache?: true} + + case Plausible.Repo.one(site_by_domain_query) do + nil -> {false, %Ecto.NoResultsError{}} + site -> {true, site} + end + end + + defp with_telemetry_metadata(props) do + Enum.into(props, %{}) + end + + defp measure_duration_with_metadata(event, fun) when is_function(fun, 0) do + {duration, {result, telemetry_metadata}} = time_it(fun) + :telemetry.execute(event, %{duration: duration}, telemetry_metadata) + result + end + + defp measure_duration(event, fun) when is_function(fun, 0) do + {duration, result} = time_it(fun) + :telemetry.execute(event, %{duration: duration}, %{}) + result + end + + defp time_it(fun) do + start = System.monotonic_time() + result = fun.() + stop = System.monotonic_time() + {stop - start, result} + end end diff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.ex index cbb705ac7..cd31863eb 100644 --- a/lib/plausible/site/cache/warmer.ex +++ b/lib/plausible/site/cache/warmer.ex @@ -19,11 +19,6 @@ defmodule Plausible.Site.Cache.Warmer do * `warmer_fn` - used for testing, a custom function to retrieve the items meant to be cached during the warm-up cycle. - On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored. - in the measurments map. The event name defaults to: - - `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]` - See tests for more comprehensive examples. """ @@ -35,9 +30,11 @@ defmodule Plausible.Site.Cache.Warmer do @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore def child_spec(opts) do + child_name = Keyword.get(opts, :child_name, {:local, __MODULE__}) + %{ id: __MODULE__, - start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]} + start: {:gen_cycle, :start_link, [child_name, __MODULE__, opts]} } end @@ -59,14 +56,10 @@ defmodule Plausible.Site.Cache.Warmer do @impl true def handle_cycle(opts) do cache_name = Keyword.fetch!(opts, :cache_name) + Logger.info("Refreshing #{cache_name} cache...") - measure_duration(telemetry_event_refresh(cache_name), fn -> - Logger.info("Refreshing #{cache_name} cache...") - - warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1) - - warmer_fn.(opts) - end) + warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.refresh_all/1) + warmer_fn.(opts) {:continue_hibernated, opts} end @@ -76,11 +69,6 @@ defmodule Plausible.Site.Cache.Warmer do {:continue, state} end - @spec telemetry_event_refresh(atom()) :: list(atom()) - def telemetry_event_refresh(cache_name) do - [:prom_ex, :plugin, :cachex, cache_name, :refresh] - end - @spec interval() :: pos_integer() def interval() do interval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval) @@ -94,18 +82,4 @@ defmodule Plausible.Site.Cache.Warmer do Enum.random(1..max_jitter) end - - defp measure_duration(event, fun) when is_function(fun, 0) do - start = System.monotonic_time() - result = fun.() - stop = System.monotonic_time() - - :telemetry.execute( - event, - %{duration: stop - start}, - %{} - ) - - result - end end diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs index 7232f9042..4b6edb7a5 100644 --- a/test/plausible/site/cache_test.exs +++ b/test/plausible/site/cache_test.exs @@ -6,115 +6,193 @@ defmodule Plausible.Site.CacheTest do import ExUnit.CaptureLog - test "cache process is started, but falls back to the database if cache is disabled" do - insert(:site, domain: "example.test") - refute Cache.enabled?() - assert Process.alive?(Process.whereis(Cache.name())) - refute Process.whereis(Cache.Warmer) - assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test") - assert Cache.size() == 0 - refute Cache.get("other.test") - end + describe "public cache interface" do + test "cache process is started, but falls back to the database if cache is disabled" do + insert(:site, domain: "example.test") + refute Cache.enabled?() + assert Process.alive?(Process.whereis(Cache.name())) + refute Process.whereis(Cache.Warmer) + assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test") + assert Cache.size() == 0 + refute Cache.get("other.test") + end - test "cache warmer process warms up the cache", %{test: test} do - test_pid = self() - opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test] + test "critical cache errors are logged and nil is returned" do + log = + capture_log(fn -> + assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil + end) - {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache" + end - assert Process.whereis(Cache.Warmer) - assert_receive {:cache_warmed, %{opts: got_opts}} - assert got_opts[:cache_name] == test - end + test "cache caches", %{test: test} do + {:ok, _} = + Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}], + strategy: :one_for_one, + name: Test.Supervisor.Cache + ) - test "cache warmer sends telemetry event", %{test: test} do - test_pid = self() + %{id: first_id} = site1 = insert(:site, domain: "site1.example.com") + _ = insert(:site, domain: "site2.example.com") - :telemetry.attach( - "test-cache-warmer", - Cache.Warmer.telemetry_event_refresh(test), - fn _event, %{duration: d}, %{}, %{} when is_integer(d) -> - send(test_pid, :telemetry_handled) - end, - %{} - ) + :ok = Cache.refresh_all(cache_name: test) - opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test] - {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + {:ok, _} = Plausible.Repo.delete(site1) - assert_receive {:cache_warmed, _} - assert_receive :telemetry_handled - end + assert Cache.size(test) == 2 - test "critical cache errors are logged and nil is returned" do - log = - capture_log(fn -> - assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil - end) + assert %Site{from_cache?: true, id: ^first_id} = + Cache.get("site1.example.com", force?: true, cache_name: test) - assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache" - end + assert %Site{from_cache?: true} = + Cache.get("site2.example.com", force?: true, cache_name: test) - test "cache warmer warms periodically with an interval", %{test: test} do - test_pid = self() - opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30] + assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test) - {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + refute Cache.get("site3.example.com", cache_name: test, force?: true) + end - assert_receive {:cache_warmed, %{at: at1}}, 100 - assert_receive {:cache_warmed, %{at: at2}}, 100 - assert_receive {:cache_warmed, %{at: at3}}, 100 + test "cache exposes hit rate", %{test: test} do + {:ok, _} = start_test_cache(test) - assert is_integer(at1) - assert is_integer(at2) - assert is_integer(at3) + insert(:site, domain: "site1.example.com") + :ok = Cache.refresh_all(cache_name: test) - assert at1 < at2 - assert at3 > at2 - end + assert Cache.hit_rate(test) == 0 + assert Cache.get("site1.example.com", force?: true, cache_name: test) + assert Cache.hit_rate(test) == 100 + refute Cache.get("nonexisting.example.com", force?: true, cache_name: test) + assert Cache.hit_rate(test) == 50 + end + + test "a single cached site can be refreshed", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain1 = "site1.example.com" + domain2 = "nonexisting.example.com" + + cache_opts = [cache_name: test, force?: true] - test "cache caches", %{test: test} do - {:ok, _} = - Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}], - strategy: :one_for_one, - name: Test.Supervisor.Cache - ) + assert Cache.get(domain1) == nil - %{id: first_id} = site1 = insert(:site, domain: "site1.example.com") - _ = insert(:site, domain: "site2.example.com") + insert(:site, domain: domain1) + + assert {:ok, %{domain: ^domain1}} = Cache.refresh_one(domain1, cache_opts) + assert %Site{domain: ^domain1} = Cache.get(domain1, cache_opts) + + assert {:ok, %Ecto.NoResultsError{}} = Cache.refresh_one(domain2, cache_opts) + assert %Ecto.NoResultsError{} = Cache.get(domain2, cache_opts) + end - :ok = Cache.prefill(cache_name: test) + test "refreshing a single site sends a telemetry event indicating record not found in the database", + %{ + test: test + } do + :ok = + start_test_cache_with_telemetry_handler(test, + event: Cache.telemetry_event_refresh(test, :one) + ) + + Cache.refresh_one("missing.example.com", force?: true, cache_name: test) + assert_receive {:telemetry_handled, %{found_in_db?: false}} + end - {:ok, _} = Plausible.Repo.delete(site1) + test "refreshing a single site sends a telemetry event indicating record found in the database", + %{ + test: test + } do + domain = "site1.example.com" + insert(:site, domain: domain) - assert Cache.size(test) == 2 + :ok = + start_test_cache_with_telemetry_handler(test, + event: Cache.telemetry_event_refresh(test, :one) + ) - assert %Site{from_cache?: true, id: ^first_id} = - Cache.get("site1.example.com", force?: true, cache_name: test) + Cache.refresh_one(domain, force?: true, cache_name: test) + assert_receive {:telemetry_handled, %{found_in_db?: true}} + end - assert %Site{from_cache?: true} = - Cache.get("site2.example.com", force?: true, cache_name: test) + test "refreshing all sites sends a telemetry event", + %{ + test: test + } do + domain = "site1.example.com" + insert(:site, domain: domain) - assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test) + :ok = + start_test_cache_with_telemetry_handler(test, + event: Cache.telemetry_event_refresh(test, :all) + ) - refute Cache.get("site3.example.com", cache_name: test, force?: true) + Cache.refresh_all(force?: true, cache_name: test) + assert_receive {:telemetry_handled, %{}} + end end - test "cache exposes hit rate", %{test: test} do - {:ok, _} = - Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}], - strategy: :one_for_one, - name: Test.Supervisor.HitRateCache - ) - - insert(:site, domain: "site1.example.com") - :ok = Cache.prefill(cache_name: test) - - assert Cache.hit_rate(test) == 0 - assert Cache.get("site1.example.com", force?: true, cache_name: test) - assert Cache.hit_rate(test) == 100 - refute Cache.get("nonexisting.example.com", force?: true, cache_name: test) - assert Cache.hit_rate(test) == 50 + describe "warming the cache" do + test "cache warmer process warms up the cache", %{test: test} do + test_pid = self() + opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test] + + {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + assert Process.whereis(Cache.Warmer) + + assert_receive {:cache_warmed, %{opts: got_opts}} + assert got_opts[:cache_name] == test + end + + test "cache warmer warms periodically with an interval", %{test: test} do + test_pid = self() + + opts = [ + force_start?: true, + warmer_fn: report_back(test_pid), + cache_name: test, + interval: 30 + ] + + {:ok, _} = start_test_warmer(opts) + + assert_receive {:cache_warmed, %{at: at1}}, 100 + assert_receive {:cache_warmed, %{at: at2}}, 100 + assert_receive {:cache_warmed, %{at: at3}}, 100 + + assert is_integer(at1) + assert is_integer(at2) + assert is_integer(at3) + + assert at1 < at2 + assert at3 > at2 + end + + test "deleted sites don't stay in cache on another refresh", %{test: test} do + {:ok, _} = start_test_cache(test) + + domain1 = "site1.example.com" + domain2 = "site2.example.com" + + site1 = insert(:site, domain: domain1) + _site2 = insert(:site, domain: domain2) + + cache_opts = [cache_name: test, force?: true] + + :ok = Cache.refresh_all(cache_opts) + + assert Cache.get(domain1, cache_opts) + assert Cache.get(domain2, cache_opts) + + Repo.delete!(site1) + + :ok = Cache.refresh_all(cache_opts) + + assert Cache.get(domain2, cache_opts) + + refute Cache.get(domain1, cache_opts) + Cache.refresh_one(domain1, cache_opts) + assert Cache.get(domain1, cache_opts) == %Ecto.NoResultsError{} + end end defp report_back(test_pid) do @@ -123,4 +201,29 @@ defmodule Plausible.Site.CacheTest do :ok end end + + defp start_test_cache(cache_name) do + %{start: {m, f, a}} = Cache.child_spec(cache_name: cache_name) + apply(m, f, a) + end + + defp start_test_warmer(opts) do + child_name_opt = {:child_name, {:local, Keyword.fetch!(opts, :cache_name)}} + %{start: {m, f, a}} = Cache.Warmer.child_spec([child_name_opt | opts]) + apply(m, f, a) + end + + defp start_test_cache_with_telemetry_handler(test, event: event) do + {:ok, _} = start_test_cache(test) + test_pid = self() + + :telemetry.attach( + "#{test}-telemetry-handler", + event, + fn ^event, %{duration: d}, metadata, _ when is_integer(d) -> + send(test_pid, {:telemetry_handled, metadata}) + end, + %{} + ) + end end