Fix: application startup, event timestamps (#2479)

### Changes

This PR:
- pushes PromEx to the bottom of supervision stack to avoid Endpoint
instrumentation failure
- ensures the site cache is ready by exposing it through the health
check endpoint
- fixes event timestamps being calculated at compile time, with
regression unit and integration tests

### Tests
- [x] Automated tests have been added
- [ ] This PR does not require tests

### Changelog
- [ ] Entry has been added to changelog
- [x] This PR does not make a user-facing change

### Documentation
- [ ] [Docs](https://github.com/plausible/docs) have been updated
- [x] This change does not need a documentation update

### Dark mode
- [ ] The UI has been tested both in dark and light mode
- [x] This PR does not change the UI
pull/2480/head r2022.11.29-25
Adam Rutkowski 4 years ago committed by GitHub
parent 356575ef78
commit 467317f24e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -21,12 +21,12 @@ defmodule Plausible.Application do
Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},
id: :cachex_sessions
),
Plausible.PromEx,
{Plausible.Site.Cache, []},
{Plausible.Site.Cache.Warmer.All, []},
{Plausible.Site.Cache.Warmer.RecentlyUpdated, []},
PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)}
{Oban, Application.get_env(:plausible, Oban)},
Plausible.PromEx
]
opts = [strategy: :one_for_one, name: Plausible.Supervisor]

@ -22,8 +22,7 @@ defmodule Plausible.Ingestion.Request do
field :props, :map
field :query_params, :map
field :timestamp, :naive_datetime,
default: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
field :timestamp, :naive_datetime
end
@type t() :: %__MODULE__{}
@ -33,7 +32,13 @@ defmodule Plausible.Ingestion.Request do
Builds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
"""
def build(%Plug.Conn{} = conn) do
changeset = Changeset.change(%__MODULE__{})
changeset =
%__MODULE__{}
|> Changeset.change()
|> Changeset.put_change(
:timestamp,
NaiveDateTime.utc_now()
)
case parse_body(conn) do
{:ok, request_body} ->

@ -55,6 +55,7 @@ defmodule Plausible.Site.Cache do
@type t() :: Site.t()
@spec name() :: atom()
def name(), do: @cache_name
@spec child_spec(Keyword.t()) :: Supervisor.child_spec()
@ -68,6 +69,22 @@ defmodule Plausible.Site.Cache do
)
end
@doc """
Ensures the cache has non-zero size unless no sites exist.
Useful for orchestrating app startup to prevent the service
going up asynchronously with an empty cache.
"""
@spec ready?(atom()) :: boolean
def ready?(cache_name \\ @cache_name) do
case size(cache_name) do
n when n > 0 ->
true
0 ->
Plausible.Repo.aggregate(Site, :count) == 0
end
end
@spec refresh_all(Keyword.t()) :: :ok
def refresh_all(opts \\ []) do
sites_by_domain_query =

@ -60,16 +60,22 @@ defmodule PlausibleWeb.Api.ExternalController do
e -> "error: #{inspect(e)}"
end
sites_cache_health =
if postgres_health == "ok" and Plausible.Site.Cache.ready?() do
"ok"
end
status =
case {postgres_health, clickhouse_health} do
{"ok", "ok"} -> 200
case {postgres_health, clickhouse_health, sites_cache_health} do
{"ok", "ok", "ok"} -> 200
_ -> 500
end
put_status(conn, status)
|> json(%{
postgres: postgres_health,
clickhouse: clickhouse_health
clickhouse: clickhouse_health,
sites_cache: sites_cache_health
})
end

@ -43,6 +43,24 @@ defmodule Plausible.Ingestion.RequestTest do
assert request.props == %{}
end
test "requests include moving timestamp" do
payload = %{
name: "pageview",
url: "http://dummy.site"
}
conn = build_conn(:post, "/api/events", payload)
assert {:ok, request1} = Request.build(conn)
:timer.sleep(1500)
assert {:ok, request2} = Request.build(conn)
ts1 = request1.timestamp
ts2 = request2.timestamp
assert NaiveDateTime.compare(ts1, ts2) == :lt
end
test "request can be built with domain" do
payload = %{
name: "pageview",

@ -53,6 +53,24 @@ defmodule Plausible.Site.CacheTest do
refute Cache.get("site3.example.com", cache_name: test, force?: true)
end
test "cache is ready when no sites exist in the db", %{test: test} do
{:ok, _} = start_test_cache(test)
assert Cache.ready?(test)
end
test "cache is not ready when sites exist in the db but cache needs refresh", %{test: test} do
{:ok, _} = start_test_cache(test)
insert(:site)
refute Cache.ready?(test)
end
test "cache is ready when refreshed", %{test: test} do
{:ok, _} = start_test_cache(test)
insert(:site)
:ok = Cache.refresh_all(cache_name: test)
assert Cache.ready?(test)
end
test "cache exposes hit rate", %{test: test} do
{:ok, _} = start_test_cache(test)

@ -62,7 +62,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 400
end
test "can send to multiple dashboards by listing multiple domains", %{
test "can send to multiple dashboards by listing multiple domains - same timestamp", %{
conn: conn,
domain: domain1
} do
@ -82,8 +82,34 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
assert response(conn, 202) == "ok"
assert get_event(domain1)
assert get_event(domain2)
assert e1 = get_event(domain1)
assert e2 = get_event(domain2)
assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :eq
end
test "timestamps differ when two events sent in a row", %{conn: conn, domain: domain} do
params = %{
domain: domain,
name: "pageview",
url: "http://gigride.live/",
referrer: "http://m.facebook.com/",
screen_width: 1440
}
conn
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
:timer.sleep(1500)
conn
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", params)
[e1, e2] = get_events(domain)
assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :gt
end
test "www. is stripped from domain", %{conn: conn, domain: domain} do
@ -1120,7 +1146,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "returns 200 OK", %{conn: conn} do
conn = get(conn, "/api/health")
assert conn.status == 200
assert payload = json_response(conn, 200)
assert payload["postgres"] == "ok"
assert payload["clickhouse"] == "ok"
assert payload["sites_cache"] == "ok"
end
end

Loading…
Cancel
Save