From 457a5584712f67b12e4e81999757add0bc22ae21 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Wed, 16 Nov 2022 10:06:23 +0100 Subject: [PATCH] Kick off sites by domain cache implementation (#2434) * Implement sites by domain caching interface + warmer * Add test * Implement hit rate interface * Add moduledocs * Fix up typespec * s/warmer/warmer_fn * Extract measure_duration/2 * Fix up typespec * Log errors and return nil on cache internal errors * Fix up non-existing cache test * Retrieve specific db columns when pre-filling the cache * Reduce the subset of fields retrieved from the DB See https://github.com/plausible/analytics/commit/63f3c6233d38581bb5789f150b8cd07dc3b0e057#r89871536 --- config/config.exs | 5 ++ config/test.exs | 3 +- lib/plausible/application.ex | 2 + lib/plausible/site.ex | 5 ++ lib/plausible/site/cache.ex | 112 +++++++++++++++++++++++++ lib/plausible/site/cache/warmer.ex | 111 +++++++++++++++++++++++++ mix.exs | 1 + mix.lock | 1 + test/plausible/site/cache_test.exs | 126 +++++++++++++++++++++++++++++ 9 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 lib/plausible/site/cache.ex create mode 100644 lib/plausible/site/cache/warmer.ex create mode 100644 test/plausible/site/cache_test.exs diff --git a/config/config.exs b/config/config.exs index 369b11b00..32456c2a7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -41,4 +41,9 @@ config :plausible, Plausible.Repo, connect_timeout: 300_000, handshake_timeout: 300_000 +config :plausible, + sites_by_domain_cache_enabled: true, + sites_by_domain_cache_refresh_interval_max_jitter: :timer.seconds(5), + sites_by_domain_cache_refresh_interval: :timer.minutes(15) + import_config "#{config_env()}.exs" diff --git a/config/test.exs b/config/test.exs index d148d4ca1..475cd355a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,4 +63,5 @@ config :geolix, config :plausible, session_timeout: 0, - http_impl: Plausible.HTTPClient.Mock + http_impl: Plausible.HTTPClient.Mock, + sites_by_domain_cache_enabled: false diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index a267567f9..b060c3449 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -21,6 +21,8 @@ defmodule Plausible.Application do Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true}, id: :cachex_sessions ), + {Plausible.Site.Cache, []}, + {Plausible.Site.Cache.Warmer, []}, PlausibleWeb.Endpoint, {Oban, Application.get_env(:plausible, Oban)}, Plausible.PromEx diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex index adfb124a5..bbe8c90b8 100644 --- a/lib/plausible/site.ex +++ b/lib/plausible/site.ex @@ -31,6 +31,11 @@ defmodule Plausible.Site do has_one :custom_domain, Plausible.Site.CustomDomain has_one :spike_notification, Plausible.Site.SpikeNotification + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Cache`. + # Use `Plausible.Repo.reload!(cached_site)` to pre-fill missing fields if + # strictly necessary. + field :from_cache?, :boolean, virtual: true, default: false + timestamps() end diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex new file mode 100644 index 000000000..c24a7c0aa --- /dev/null +++ b/lib/plausible/site/cache.ex @@ -0,0 +1,112 @@ +defmodule Plausible.Site.Cache do + @moduledoc """ + A sites by domain caching interface. + + Serves as a thin wrapper around Cachex, but the underlying + implementation can be transparently swapped. + + Even though the Cachex process is started, cache access is disabled + during tests via the `:sites_by_domain_cache_enabled` application env key. + This can be overriden on case by case basis, using the child specs options. + + When Cache is disabled via application env, the `get/1` function + falls back to pure database lookups. This should help with introducing + cached lookups in existing code, so that no existing tests should break. + + To differentiate cached Site structs from those retrieved directly from the + database, a virtual schema field `from_cache?: true` is set. + This indicates the `Plausible.Site` struct is incomplete in comparison to its + database counterpart -- to spare bandwidth and query execution time, + only selected database columns are retrieved and cached. + + The `@cached_schema_fields` attribute defines the list of DB columns + queried on cache pre-fill. + + Also see tests for more comprehensive examples. + """ + require Logger + + import Ecto.Query + + alias Plausible.Site + + @cache_name :sites_by_domain + + @cached_schema_fields ~w( + id + domain + ingest_rate_limit_scale_seconds + ingest_rate_limit_threshold + )a + + def name(), do: @cache_name + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() + def child_spec(opts) do + cache_name = Keyword.get(opts, :cache_name, @cache_name) + child_id = Keyword.get(opts, :child_id, :cachex_sites) + + Supervisor.child_spec( + {Cachex, name: cache_name, limit: nil, stats: true}, + id: child_id + ) + end + + @spec prefill(Keyword.t()) :: :ok + def prefill(opts) do + cache_name = Keyword.fetch!(opts, :cache_name) + + sites_by_domain_query = + from s in Site, + select: { + s.domain, + %{struct(s, ^@cached_schema_fields) | from_cache?: true} + } + + sites_by_domain = Plausible.Repo.all(sites_by_domain_query) + + true = Cachex.put_many!(cache_name, sites_by_domain) + :ok + end + + @spec size() :: non_neg_integer() + def size(cache_name \\ @cache_name) do + {:ok, size} = Cachex.size(cache_name) + size + end + + @spec hit_rate() :: number() + def hit_rate(cache_name \\ @cache_name) do + {:ok, stats} = Cachex.stats(cache_name) + Map.get(stats, :hit_rate, 0) + end + + @spec get(String.t(), Keyword.t()) :: nil | Site.t() + def get(domain, opts \\ []) do + cache_name = Keyword.get(opts, :cache_name, @cache_name) + force? = Keyword.get(opts, :force?, false) + + if enabled?() or force? do + case Cachex.get(cache_name, domain) do + {:ok, nil} -> + nil + + {:ok, site} -> + site + + {:error, e} -> + Logger.error( + "Error retrieving '#{domain}' from '#{inspect(cache_name)}': #{inspect(e)}" + ) + + nil + end + else + Plausible.Sites.get_by_domain(domain) + end + end + + def enabled?() do + Application.fetch_env!(:plausible, :sites_by_domain_cache_enabled) == true + end +end diff --git a/lib/plausible/site/cache/warmer.ex b/lib/plausible/site/cache/warmer.ex new file mode 100644 index 000000000..cbb705ac7 --- /dev/null +++ b/lib/plausible/site/cache/warmer.ex @@ -0,0 +1,111 @@ +defmodule Plausible.Site.Cache.Warmer do + @moduledoc """ + A periodic cache warmer. + Queries all Sites from Postgres, every `interval` and pre-populates the cache. + After each run the process is hibernated, triggering garbage collection. + + Currently Cachex is used, but the underlying implementation can be transparently swapped. + + Child specification options available: + + * `interval` - the number of milliseconds for each warm-up cycle, defaults + to `:sites_by_domain_cache_refresh_interval` application env value + with random jitter added, for which the maximum is stored under + `:sites_by_domain_cache_refresh_interval_max_jitter` key. + * `cache_name` - defaults to Cache.name() but can be overriden for testing + * `force_start?` - enforcess process startup for testing, even if it's barred + by `Cache.enabled?`. This is useful for avoiding issues with DB ownership + and async tests. + * `warmer_fn` - used for testing, a custom function to retrieve the items meant + to be cached during the warm-up cycle. + + On each warm-up cycle, a telemetry event is emitted with warm-up `duration` stored. + in the measurments map. The event name defaults to: + + `[:prom_ex, :plugin, :cachex, Cache.name(), :refresh]` + + See tests for more comprehensive examples. + """ + + @behaviour :gen_cycle + + require Logger + + alias Plausible.Site.Cache + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore + def child_spec(opts) do + %{ + id: __MODULE__, + start: {:gen_cycle, :start_link, [{:local, __MODULE__}, __MODULE__, opts]} + } + end + + @impl true + def init_cycle(opts) do + interval = Keyword.get(opts, :interval, interval()) + force_start? = Keyword.get(opts, :force_start?, false) + cache_name = Keyword.get(opts, :cache_name, Cache.name()) + + if Cache.enabled?() or force_start? do + Logger.info("Initializing #{__MODULE__} with interval #{interval}") + + {:ok, {interval, Keyword.put(opts, :cache_name, cache_name)}} + else + :ignore + end + end + + @impl true + def handle_cycle(opts) do + cache_name = Keyword.fetch!(opts, :cache_name) + + measure_duration(telemetry_event_refresh(cache_name), fn -> + Logger.info("Refreshing #{cache_name} cache...") + + warmer_fn = Keyword.get(opts, :warmer_fn, &Cache.prefill/1) + + warmer_fn.(opts) + end) + + {:continue_hibernated, opts} + end + + @impl true + def handle_info(_msg, state) do + {:continue, state} + end + + @spec telemetry_event_refresh(atom()) :: list(atom()) + def telemetry_event_refresh(cache_name) do + [:prom_ex, :plugin, :cachex, cache_name, :refresh] + end + + @spec interval() :: pos_integer() + def interval() do + interval = Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval) + + interval + jitter() + end + + defp jitter() do + max_jitter = + Application.fetch_env!(:plausible, :sites_by_domain_cache_refresh_interval_max_jitter) + + Enum.random(1..max_jitter) + end + + defp measure_duration(event, fun) when is_function(fun, 0) do + start = System.monotonic_time() + result = fun.() + stop = System.monotonic_time() + + :telemetry.execute( + event, + %{duration: stop - start}, + %{} + ) + + result + end +end diff --git a/mix.exs b/mix.exs index 1eeb1d0dd..a78b93987 100644 --- a/mix.exs +++ b/mix.exs @@ -80,6 +80,7 @@ defmodule Plausible.MixProject do {:floki, "~> 0.32.0", only: :test}, {:fun_with_flags, "~> 1.9.0"}, {:fun_with_flags_ui, "~> 0.8"}, + {:gen_cycle, "~> 1.0"}, {:geolix, "~> 2.0"}, {:geolix_adapter_mmdb2, "~> 0.6.0"}, {:hackney, "~> 1.8"}, diff --git a/mix.lock b/mix.lock index 635786835..473e80adc 100644 --- a/mix.lock +++ b/mix.lock @@ -47,6 +47,7 @@ "floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"}, "fun_with_flags": {:hex, :fun_with_flags, "1.9.0", "0be8692727623af0c00e353f7bdcc9934dd6e1f87798ca6bfec9e739d42b63e6", [:mix], [{:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: true]}, {:redix, "~> 1.0", [hex: :redix, repo: "hexpm", optional: true]}], "hexpm", "8145dc009d549389f1b1915a715fb7e357fcd8fd7b91c74f04aae74912eef27a"}, "fun_with_flags_ui": {:hex, :fun_with_flags_ui, "0.8.0", "70587e344ba2035516a639e7bd8cb2ce8d54ee6c5f5dfd3e4e6f6a776e14ac1d", [:mix], [{:cowboy, ">= 2.0.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:fun_with_flags, "~> 1.8", [hex: :fun_with_flags, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "95e1e5afae77e259a4a43f930f3da97ff3c388a72d4622f7848cb7ee34de6338"}, + "gen_cycle": {:hex, :gen_cycle, "1.0.3", "ee0117b9c65814f58827abfe84967d4cb5380db0fbac992358e5b669b3ebcbcb", [:rebar3], [], "hexpm", "fffdadf7fb73de75e3429eb8a6982768e7240803561942602350b689991749e4"}, "gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"}, "geolix": {:hex, :geolix, "2.0.0", "7e65764bedfc98d08a3ddb24c417657c9d438eff163280b45fbb7de289626acd", [:mix], [], "hexpm", "8742bf588ed0bb7def2c443204d09d355990846c6efdff96ded66aac24c301df"}, "geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.6.0", "6ab9dbf6ea395817aa1fd2597be25d0dda1853c7f664e62716e47728d18bc4f9", [:mix], [{:geolix, "~> 2.0", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "06ff962feae8a310cffdf86b74bfcda6e2d0dccb439bb1f62df2b657b1c0269b"}, diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs new file mode 100644 index 000000000..7232f9042 --- /dev/null +++ b/test/plausible/site/cache_test.exs @@ -0,0 +1,126 @@ +defmodule Plausible.Site.CacheTest do + use Plausible.DataCase, async: true + + alias Plausible.Site + alias Plausible.Site.Cache + + import ExUnit.CaptureLog + + test "cache process is started, but falls back to the database if cache is disabled" do + insert(:site, domain: "example.test") + refute Cache.enabled?() + assert Process.alive?(Process.whereis(Cache.name())) + refute Process.whereis(Cache.Warmer) + assert %Site{domain: "example.test", from_cache?: false} = Cache.get("example.test") + assert Cache.size() == 0 + refute Cache.get("other.test") + end + + test "cache warmer process warms up the cache", %{test: test} do + test_pid = self() + opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test] + + {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + + assert Process.whereis(Cache.Warmer) + assert_receive {:cache_warmed, %{opts: got_opts}} + assert got_opts[:cache_name] == test + end + + test "cache warmer sends telemetry event", %{test: test} do + test_pid = self() + + :telemetry.attach( + "test-cache-warmer", + Cache.Warmer.telemetry_event_refresh(test), + fn _event, %{duration: d}, %{}, %{} when is_integer(d) -> + send(test_pid, :telemetry_handled) + end, + %{} + ) + + opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test] + {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + + assert_receive {:cache_warmed, _} + assert_receive :telemetry_handled + end + + test "critical cache errors are logged and nil is returned" do + log = + capture_log(fn -> + assert Cache.get("key", force?: true, cache_name: NonExistingCache) == nil + end) + + assert log =~ "Error retrieving 'key' from 'NonExistingCache': :no_cache" + end + + test "cache warmer warms periodically with an interval", %{test: test} do + test_pid = self() + opts = [force_start?: true, warmer_fn: report_back(test_pid), cache_name: test, interval: 30] + + {:ok, _} = Supervisor.start_link([{Cache.Warmer, opts}], strategy: :one_for_one, name: test) + + assert_receive {:cache_warmed, %{at: at1}}, 100 + assert_receive {:cache_warmed, %{at: at2}}, 100 + assert_receive {:cache_warmed, %{at: at3}}, 100 + + assert is_integer(at1) + assert is_integer(at2) + assert is_integer(at3) + + assert at1 < at2 + assert at3 > at2 + end + + test "cache caches", %{test: test} do + {:ok, _} = + Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}], + strategy: :one_for_one, + name: Test.Supervisor.Cache + ) + + %{id: first_id} = site1 = insert(:site, domain: "site1.example.com") + _ = insert(:site, domain: "site2.example.com") + + :ok = Cache.prefill(cache_name: test) + + {:ok, _} = Plausible.Repo.delete(site1) + + assert Cache.size(test) == 2 + + assert %Site{from_cache?: true, id: ^first_id} = + Cache.get("site1.example.com", force?: true, cache_name: test) + + assert %Site{from_cache?: true} = + Cache.get("site2.example.com", force?: true, cache_name: test) + + assert %Site{from_cache?: false} = Cache.get("site2.example.com", cache_name: test) + + refute Cache.get("site3.example.com", cache_name: test, force?: true) + end + + test "cache exposes hit rate", %{test: test} do + {:ok, _} = + Supervisor.start_link([{Cache, [cache_name: test, child_id: :test_cache_caches_id]}], + strategy: :one_for_one, + name: Test.Supervisor.HitRateCache + ) + + insert(:site, domain: "site1.example.com") + :ok = Cache.prefill(cache_name: test) + + assert Cache.hit_rate(test) == 0 + assert Cache.get("site1.example.com", force?: true, cache_name: test) + assert Cache.hit_rate(test) == 100 + refute Cache.get("nonexisting.example.com", force?: true, cache_name: test) + assert Cache.hit_rate(test) == 50 + end + + defp report_back(test_pid) do + fn opts -> + send(test_pid, {:cache_warmed, %{at: System.monotonic_time(), opts: opts}}) + :ok + end + end +end