Compare commits

...

7 Commits

@ -23,7 +23,9 @@ defmodule Plausible.Application do
), ),
PlausibleWeb.Endpoint, PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)}, {Oban, Application.get_env(:plausible, Oban)},
Plausible.PromEx Plausible.PromEx,
Plausible.Ingestion.Session.Registry.child_spec(),
Plausible.Ingestion.Session.DynamicSupervisor
] ]
opts = [strategy: :one_for_one, name: Plausible.Supervisor] opts = [strategy: :one_for_one, name: Plausible.Supervisor]

@ -2,6 +2,8 @@ defmodule Plausible.ClickhouseEvent do
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset import Ecto.Changeset
@type t() :: %__MODULE__{}
@primary_key false @primary_key false
schema "events" do schema "events" do
field :name, :string field :name, :string

@ -0,0 +1,18 @@
defmodule Plausible.Ingestion do
alias Plausible.Ingestion.{Event, Session, Request}
@spec from_requests([Request.t()]) :: :ok
@doc """
Ingests a list of requests. If a session already exists for the request, it uses the existing
one, otherwise it creates a new session. Events and sessions are buffered before being inserted
into Clickhouse.
"""
def from_requests(requests) do
Enum.each(requests, fn request ->
user_id = Event.get_user_id(request)
pid = Session.DynamicSupervisor.find_or_spawn(request.domain, user_id)
Session.Actor.send_event(pid, request)
end)
end
end

@ -1,15 +1,18 @@
defmodule Plausible.Ingestion.Event do defmodule Plausible.Ingestion.Event do
require OpenTelemetry.Tracer, as: Tracer require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Ingestion.{Request, CityOverrides} alias Plausible.Ingestion.{Request, CityOverrides}
use OpenTelemetryDecorator
@spec build_and_buffer(Request.t()) :: :ok | :skip | {:error, Ecto.Changeset.t()} @decorate trace("ingestion.build_event_struct")
@spec build(Request.t()) ::
{:ok, Plausible.ClickhouseEvent.t()} | :skip | {:error, Ecto.Changeset.t()}
@doc """ @doc """
Builds events from %Plausible.Ingestion.Request{} and adds them to Plausible.Event.WriteBuffer. Builds events from %Plausible.Ingestion.Request{}. This function reads geolocation data and
This function reads geolocation data and parses the user agent string. Returns :skip if the parses the user agent string. Returns :skip if the request is identified as spam, or blocked.
request is identified as spam, or blocked.
""" """
def build_and_buffer(%Request{} = request) do def build(%Request{} = request) do
with :ok <- spam_or_blocked?(request), with :ok <- skip_if_spam(request),
:ok <- skip_if_blocked(request),
salts <- Plausible.Session.Salts.fetch(), salts <- Plausible.Session.Salts.fetch(),
event <- Map.new(), event <- Map.new(),
%{} = event <- put_user_agent(event, request), %{} = event <- put_user_agent(event, request),
@ -18,23 +21,35 @@ defmodule Plausible.Ingestion.Event do
%{} = event <- put_geolocation(event, request), %{} = event <- put_geolocation(event, request),
%{} = event <- put_screen_size(event, request), %{} = event <- put_screen_size(event, request),
%{} = event <- put_props(event, request), %{} = event <- put_props(event, request),
events when is_list(events) <- map_domains(event, request), %{} = event <- put_user_id(event, request, salts),
events when is_list(events) <- put_user_id(events, request, salts), {:ok, event} <- validate_event(event),
{:ok, events} <- validate_events(events), do: {:ok, event}
events when is_list(events) <- register_session(events, request, salts) do end
Enum.each(events, &Plausible.Event.WriteBuffer.insert/1)
end @decorate trace("ingestion.generate_user_id")
def get_user_id(%Request{domain: domain} = request) do
salt = Plausible.Session.Salts.fetch().current
user_agent = request.user_agent || ""
root_domain = request.uri |> Request.sanitize_hostname() |> get_root_domain()
SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain)
end end
defp put_basic_info(%{} = event, %Request{} = request) do defp put_basic_info(%{} = event, %Request{} = request) do
uri = request.url && URI.parse(request.url) host =
host = if uri && uri.host == "", do: "(none)", else: uri && uri.host if request.uri && request.uri.host == "" do
"(none)"
else
request.uri && request.uri.host
end
event event
|> Map.put(:domain, request.domain)
|> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)) |> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))
|> Map.put(:name, request.event_name) |> Map.put(:name, request.event_name)
|> Map.put(:hostname, strip_www(host)) |> Map.put(:hostname, Request.sanitize_hostname(host))
|> Map.put(:pathname, get_pathname(uri, request.hash_mode)) |> Map.put(:pathname, get_pathname(request.uri, request.hash_mode))
end end
defp get_pathname(_uri = nil, _hash_mode), do: "/" defp get_pathname(_uri = nil, _hash_mode), do: "/"
@ -63,8 +78,7 @@ defmodule Plausible.Ingestion.Event do
end end
defp put_referrer(%{} = event, %Request{} = request) do defp put_referrer(%{} = event, %Request{} = request) do
uri = request.url && URI.parse(request.url) ref = parse_referrer(request.uri, request.referrer)
ref = parse_referrer(uri, request.referrer)
event event
|> Map.put(:utm_medium, request.query_params["utm_medium"]) |> Map.put(:utm_medium, request.query_params["utm_medium"])
@ -81,7 +95,8 @@ defmodule Plausible.Ingestion.Event do
defp parse_referrer(uri, referrer_str) do defp parse_referrer(uri, referrer_str) do
referrer_uri = URI.parse(referrer_str) referrer_uri = URI.parse(referrer_str)
if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do if Request.sanitize_hostname(referrer_uri.host) !== Request.sanitize_hostname(uri.host) &&
referrer_uri.host !== "localhost" do
RefInspector.parse(referrer_str) RefInspector.parse(referrer_str)
end end
end end
@ -127,13 +142,12 @@ defmodule Plausible.Ingestion.Event do
end end
end end
@decorate trace("ingestion.parse_user_agent")
defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do
Tracer.with_span "parse_user_agent" do case Cachex.fetch(:user_agents, user_agent, &UAInspector.parse/1) do
case Cachex.fetch(:user_agents, user_agent, &UAInspector.parse/1) do {:ok, user_agent} -> user_agent
{:ok, user_agent} -> user_agent {:commit, user_agent} -> user_agent
{:commit, user_agent} -> user_agent _ -> nil
_ -> nil
end
end end
end end
@ -205,78 +219,48 @@ defmodule Plausible.Ingestion.Event do
Map.put(event, :screen_size, screen_width) Map.put(event, :screen_size, screen_width)
end end
@decorate trace("ingestion.parse_visitor_location")
defp put_geolocation(%{} = event, %Request{} = request) do defp put_geolocation(%{} = event, %Request{} = request) do
Tracer.with_span "parse_visitor_location" do result = Geolix.lookup(request.remote_ip, where: :geolocation)
result = Geolix.lookup(request.remote_ip, where: :geolocation)
country_code = country_code =
get_in(result, [:country, :iso_code]) get_in(result, [:country, :iso_code])
|> ignore_unknown_country() |> ignore_unknown_country()
city_geoname_id = get_in(result, [:city, :geoname_id]) city_geoname_id = get_in(result, [:city, :geoname_id])
city_geoname_id = Map.get(CityOverrides.get(), city_geoname_id, city_geoname_id) city_geoname_id = Map.get(CityOverrides.get(), city_geoname_id, city_geoname_id)
subdivision1_code = subdivision1_code =
case result do case result do
%{subdivisions: [%{iso_code: iso_code} | _rest]} -> %{subdivisions: [%{iso_code: iso_code} | _rest]} ->
country_code <> "-" <> iso_code country_code <> "-" <> iso_code
_ -> _ ->
"" ""
end end
subdivision2_code = subdivision2_code =
case result do case result do
%{subdivisions: [_first, %{iso_code: iso_code} | _rest]} -> %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->
country_code <> "-" <> iso_code country_code <> "-" <> iso_code
_ -> _ ->
"" ""
end end
event event
|> Map.put(:country_code, country_code) |> Map.put(:country_code, country_code)
|> Map.put(:subdivision1_code, subdivision1_code) |> Map.put(:subdivision1_code, subdivision1_code)
|> Map.put(:subdivision2_code, subdivision2_code) |> Map.put(:subdivision2_code, subdivision2_code)
|> Map.put(:city_geoname_id, city_geoname_id) |> Map.put(:city_geoname_id, city_geoname_id)
end
end end
defp ignore_unknown_country("ZZ"), do: nil defp ignore_unknown_country("ZZ"), do: nil
defp ignore_unknown_country(country), do: country defp ignore_unknown_country(country), do: country
defp map_domains(%{} = event, %Request{} = request) do defp put_user_id(%{} = event, %Request{} = request, salts) do
domains = user_id = generate_user_id(request, event.domain, event.hostname, salts.current)
if request.domain do Map.put(event, :user_id, user_id)
String.split(request.domain, ",")
|> Enum.map(&String.trim/1)
|> Enum.map(&strip_www/1)
else
uri = request.url && URI.parse(request.url)
[strip_www(uri && uri.host)]
end
for domain <- domains, do: Map.put(event, :domain, domain)
end
defp put_user_id(events, %Request{} = request, salts) do
for %{} = event <- events do
user_id = generate_user_id(request, event.domain, event.hostname, salts.current)
Map.put(event, :user_id, user_id)
end
end
defp register_session(events, %Request{} = request, salts) do
for %Plausible.ClickhouseEvent{} = event <- events do
previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous)
session_id =
Tracer.with_span "cache_store_event" do
Plausible.Session.CacheStore.on_event(event, previous_user_id)
end
Map.put(event, :session_id, session_id)
end
end end
defp generate_user_id(request, domain, hostname, salt) do defp generate_user_id(request, domain, hostname, salt) do
@ -304,7 +288,18 @@ defmodule Plausible.Ingestion.Event do
end end
end end
defp spam_or_blocked?(%Request{} = request) do defp skip_if_spam(%Request{} = request) do
if request.referrer &&
URI.parse(request.referrer).host
|> Request.sanitize_hostname()
|> ReferrerBlocklist.is_spammer?() do
:skip
else
:ok
end
end
defp skip_if_blocked(%Request{} = request) do
cond do cond do
request.domain in Application.get_env(:plausible, :domain_blacklist) -> request.domain in Application.get_env(:plausible, :domain_blacklist) ->
:skip :skip
@ -313,32 +308,18 @@ defmodule Plausible.Ingestion.Event do
Tracer.set_attribute("blocked_by_flag", true) Tracer.set_attribute("blocked_by_flag", true)
:skip :skip
request.referrer &&
URI.parse(request.referrer).host |> strip_www() |> ReferrerBlocklist.is_spammer?() ->
:skip
true -> true ->
:ok :ok
end end
end end
defp validate_events(events) do defp validate_event(%{} = attrs) do
Enum.reduce_while(events, {:ok, []}, fn %{} = attrs, {:ok, acc} -> attrs
attrs |> Plausible.ClickhouseEvent.new()
|> Plausible.ClickhouseEvent.new() |> Ecto.Changeset.apply_action(nil)
|> Ecto.Changeset.apply_action(nil) |> case do
|> case do {:ok, event} -> {:ok, event}
{:ok, event} -> {:cont, {:ok, [event | acc]}} {:error, changeset} -> {:error, changeset}
{:error, changeset} -> {:halt, {:error, changeset}}
end
end)
end
defp strip_www(hostname) do
if hostname do
String.replace_prefix(hostname, "www.", "")
else
nil
end end
end end
end end

@ -3,11 +3,13 @@ defmodule Plausible.Ingestion.Request do
The %Plausible.Ingestion.Request{} struct stores all needed fields to create an event downstream. The %Plausible.Ingestion.Request{} struct stores all needed fields to create an event downstream.
""" """
use OpenTelemetryDecorator
defstruct [ defstruct [
:remote_ip, :remote_ip,
:user_agent, :user_agent,
:event_name, :event_name,
:url, :uri,
:referrer, :referrer,
:domain, :domain,
:screen_width, :screen_width,
@ -20,7 +22,7 @@ defmodule Plausible.Ingestion.Request do
remote_ip: String.t() | nil, remote_ip: String.t() | nil,
user_agent: String.t() | nil, user_agent: String.t() | nil,
event_name: term(), event_name: term(),
url: term(), uri: URI.t() | nil,
referrer: term(), referrer: term(),
domain: term(), domain: term(),
screen_width: term(), screen_width: term(),
@ -29,17 +31,20 @@ defmodule Plausible.Ingestion.Request do
query_params: map() query_params: map()
} }
@spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json} @decorate trace("ingestion.build_request_struct")
@spec build(Plug.Conn.t()) :: {:ok, [t()]} | {:error, :invalid_json}
@doc """ @doc """
Builds a %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. Builds a list of %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
""" """
def build(%Plug.Conn{} = conn) do def build(%Plug.Conn{} = conn) do
with {:ok, request_body} <- parse_body(conn) do with {:ok, request_body} <- parse_body(conn) do
%__MODULE__{} %__MODULE__{}
|> Map.put(:remote_ip, PlausibleWeb.RemoteIp.get(conn)) |> Map.put(:remote_ip, PlausibleWeb.RemoteIp.get(conn))
|> put_uri(request_body)
|> put_user_agent(conn) |> put_user_agent(conn)
|> put_request_params(request_body) |> put_request_params(request_body)
|> put_query_params() |> put_query_params()
|> map_domains(request_body)
|> then(&{:ok, &1}) |> then(&{:ok, &1})
end end
end end
@ -63,15 +68,36 @@ defmodule Plausible.Ingestion.Request do
%__MODULE__{ %__MODULE__{
request request
| event_name: request_body["n"] || request_body["name"], | event_name: request_body["n"] || request_body["name"],
url: request_body["u"] || request_body["url"],
referrer: request_body["r"] || request_body["referrer"], referrer: request_body["r"] || request_body["referrer"],
domain: request_body["d"] || request_body["domain"],
screen_width: request_body["w"] || request_body["screen_width"], screen_width: request_body["w"] || request_body["screen_width"],
hash_mode: request_body["h"] || request_body["hashMode"], hash_mode: request_body["h"] || request_body["hashMode"],
props: parse_props(request_body) props: parse_props(request_body)
} }
end end
defp map_domains(%__MODULE__{} = request, %{} = request_body) do
domains =
if raw = request_body["d"] || request_body["domain"] do
raw
|> String.split(",")
|> Enum.map(&sanitize_hostname/1)
else
request.uri
|> sanitize_hostname()
|> List.wrap()
end
for domain <- domains, do: %__MODULE__{request | domain: domain}
end
defp put_uri(%__MODULE__{} = request, %{} = request_body) do
if url = request_body["u"] || request_body["url"] do
%__MODULE__{request | uri: URI.parse(url)}
else
request
end
end
defp parse_props(%{} = request_body) do defp parse_props(%{} = request_body) do
raw_props = raw_props =
request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"] request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]
@ -111,13 +137,13 @@ defmodule Plausible.Ingestion.Request do
end end
end end
defp put_query_params(%__MODULE__{url: url} = request) do defp put_query_params(%__MODULE__{} = request) do
with url when is_binary(url) <- url, case request do
%URI{query: query} when is_binary(query) <- URI.parse(url), %__MODULE__{uri: %URI{query: query}} when is_binary(query) ->
%{} = query_params <- URI.decode_query(query) do %__MODULE__{request | query_params: URI.decode_query(query)}
Map.put(request, :query_params, query_params)
else _any ->
_any -> request request
end end
end end
@ -129,4 +155,21 @@ defmodule Plausible.Ingestion.Request do
%__MODULE__{request | user_agent: user_agent} %__MODULE__{request | user_agent: user_agent}
end end
@doc """
Removes the "www" part of a hostname.
"""
def sanitize_hostname(%URI{host: hostname}) do
sanitize_hostname(hostname)
end
def sanitize_hostname(hostname) when is_binary(hostname) do
hostname
|> String.trim()
|> String.replace_prefix("www.", "")
end
def sanitize_hostname(nil) do
nil
end
end end

@ -0,0 +1,106 @@
defmodule Plausible.Ingestion.Session do
@spec upsert_from_event(Plausible.ClickhouseSession.t() | nil, Plausible.ClickhouseEvent.t()) ::
Plausible.ClickhouseSession.t()
@doc """
Builds and buffers a new session if it does not exist, or updates the existing one from an
event.
"""
def upsert_from_event(session_or_nil, event)
def upsert_from_event(nil = _session, event) do
session = %Plausible.ClickhouseSession{
sign: 1,
session_id: Plausible.ClickhouseSession.random_uint64(),
hostname: event.hostname,
domain: event.domain,
user_id: event.user_id,
entry_page: event.pathname,
exit_page: event.pathname,
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),
events: 1,
referrer: event.referrer,
referrer_source: event.referrer_source,
utm_medium: event.utm_medium,
utm_source: event.utm_source,
utm_campaign: event.utm_campaign,
utm_content: event.utm_content,
utm_term: event.utm_term,
country_code: event.country_code,
subdivision1_code: event.subdivision1_code,
subdivision2_code: event.subdivision2_code,
city_geoname_id: event.city_geoname_id,
screen_size: event.screen_size,
operating_system: event.operating_system,
operating_system_version: event.operating_system_version,
browser: event.browser,
browser_version: event.browser_version,
timestamp: event.timestamp,
start: event.timestamp,
"entry_meta.key": Map.get(event, :"meta.key"),
"entry_meta.value": Map.get(event, :"meta.value")
}
{:ok, [session]} = Plausible.Session.WriteBuffer.insert([session])
session
end
def upsert_from_event(old_session, event) do
updated = %{
old_session
| user_id: event.user_id,
timestamp: event.timestamp,
exit_page: event.pathname,
is_bounce: false,
duration: Timex.diff(event.timestamp, old_session.start, :second) |> abs,
pageviews:
if(event.name == "pageview", do: old_session.pageviews + 1, else: old_session.pageviews),
country_code:
if(old_session.country_code == "",
do: event.country_code,
else: old_session.country_code
),
subdivision1_code:
if(old_session.subdivision1_code == "",
do: event.subdivision1_code,
else: old_session.subdivision1_code
),
subdivision2_code:
if(old_session.subdivision2_code == "",
do: event.subdivision2_code,
else: old_session.subdivision2_code
),
city_geoname_id:
if(old_session.city_geoname_id == 0,
do: event.city_geoname_id,
else: old_session.city_geoname_id
),
operating_system:
if(old_session.operating_system == "",
do: event.operating_system,
else: old_session.operating_system
),
operating_system_version:
if(old_session.operating_system_version == "",
do: event.operating_system_version,
else: old_session.operating_system_version
),
browser: if(old_session.browser == "", do: event.browser, else: old_session.browser),
browser_version:
if(old_session.browser_version == "",
do: event.browser_version,
else: old_session.browser_version
),
screen_size:
if(old_session.screen_size == "", do: event.screen_size, else: old_session.screen_size),
events: old_session.events + 1
}
{:ok, [updated_session, _old_session]} =
Plausible.Session.WriteBuffer.insert([%{updated | sign: 1}, %{old_session | sign: -1}])
updated_session
end
end

@ -0,0 +1,60 @@
defmodule Plausible.Ingestion.Session.Actor do
@moduledoc """
The Session actor is a short-lived process spawned for each incoming session. The process self
destructs after 30 minutes from the first event.
"""
alias Plausible.{Ingestion, Ingestion.Session}
require Logger
use GenServer, restart: :transient, shutdown: :timer.minutes(30)
use OpenTelemetryDecorator
@spec send_event(pid(), map()) :: :ok
def send_event(pid, request_or_event), do: GenServer.cast(pid, {:send_event, request_or_event})
defmodule State do
defstruct [:domain, :user_id, :session]
@type t() :: %__MODULE__{
domain: String.t(),
user_id: integer(),
session: %Plausible.ClickhouseSession{} | nil
}
end
def start_link({domain, user_id}) do
process_name = {:via, Registry, {Ingestion.Session.Registry, {domain, user_id}}}
GenServer.start_link(__MODULE__, {domain, user_id}, name: process_name)
end
@impl true
def init({domain, user_id}) do
Logger.debug("Ingestion: Starting new session for #{domain}@#{user_id}")
{:ok, %State{domain: domain, user_id: user_id}}
end
@impl true
@decorate trace("ingestion.session_send_event")
def handle_cast({:send_event, request_or_event}, %State{} = state) do
Logger.debug("Ingestion: Processing new event for session #{state.domain}@#{state.user_id}")
with {:ok, event} <- build_event(request_or_event),
session <- Session.upsert_from_event(state.session, event),
event <- %Plausible.ClickhouseEvent{event | session_id: session.session_id},
{:ok, _event} <- Plausible.Event.WriteBuffer.insert(event) do
{:noreply, %State{state | session: session}}
else
:skip ->
Logger.debug("Ingestion: Skipping spam/bot event")
{:noreply, state}
{:error, changeset} ->
Logger.error("Ingestion: Failed to insert event. Reason: #{inspect(changeset)}")
{:noreply, state}
end
end
defp build_event(%Ingestion.Request{} = request), do: Ingestion.Event.build(request)
defp build_event(%Plausible.ClickhouseEvent{} = event), do: {:ok, event}
end

@ -0,0 +1,35 @@
defmodule Plausible.Ingestion.Session.DynamicSupervisor do
alias Plausible.Ingestion.Session
use DynamicSupervisor
use OpenTelemetryDecorator
def start_link(_init_arg) do
DynamicSupervisor.start_link(__MODULE__, :ok, name: __MODULE__)
end
@impl true
def init(:ok) do
DynamicSupervisor.init(strategy: :one_for_one)
end
@decorate trace("ingestion.find_or_spawn_session_event")
def find_or_spawn(domain, user_id) do
case Registry.lookup(Session.Registry, {domain, user_id}) do
[{pid, _}] -> pid
[] -> spawn_process(domain, user_id)
end
end
defp spawn_process(domain, user_id) do
child_spec = %{
id: Session.Actor,
start: {Session.Actor, :start_link, [{domain, user_id}]},
restart: :transient
}
case DynamicSupervisor.start_child(__MODULE__, child_spec) do
{:ok, pid} -> pid
{:error, {:already_started, pid}} -> pid
end
end
end

@ -0,0 +1,5 @@
defmodule Plausible.Ingestion.Session.Registry do
def child_spec do
Registry.child_spec(keys: :unique, name: __MODULE__, partitions: System.schedulers_online())
end
end

@ -1,130 +0,0 @@
defmodule Plausible.Session.CacheStore do
require Logger
alias Plausible.Session.WriteBuffer
def on_event(event, prev_user_id, buffer \\ WriteBuffer) do
found_session = find_session(event, event.user_id) || find_session(event, prev_user_id)
session =
if found_session do
updated_session = update_session(found_session, event)
buffer.insert([%{updated_session | sign: 1}, %{found_session | sign: -1}])
persist_session(updated_session)
else
new_session = new_session_from_event(event)
buffer.insert([new_session])
persist_session(new_session)
end
session.session_id
end
defp find_session(_domain, nil), do: nil
defp find_session(event, user_id) do
case Cachex.get(:sessions, {event.domain, user_id}) do
{:ok, nil} ->
nil
{:ok, session} ->
if Timex.diff(event.timestamp, session.timestamp, :minutes) <= 30 do
session
end
{:error, e} ->
Sentry.capture_message("Cachex error", extra: %{error: e})
nil
end
end
defp persist_session(session) do
key = {session.domain, session.user_id}
Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))
session
end
defp update_session(session, event) do
%{
session
| user_id: event.user_id,
timestamp: event.timestamp,
exit_page: event.pathname,
is_bounce: false,
duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews:
if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews),
country_code:
if(session.country_code == "", do: event.country_code, else: session.country_code),
subdivision1_code:
if(session.subdivision1_code == "",
do: event.subdivision1_code,
else: session.subdivision1_code
),
subdivision2_code:
if(session.subdivision2_code == "",
do: event.subdivision2_code,
else: session.subdivision2_code
),
city_geoname_id:
if(session.city_geoname_id == 0,
do: event.city_geoname_id,
else: session.city_geoname_id
),
operating_system:
if(session.operating_system == "",
do: event.operating_system,
else: session.operating_system
),
operating_system_version:
if(session.operating_system_version == "",
do: event.operating_system_version,
else: session.operating_system_version
),
browser: if(session.browser == "", do: event.browser, else: session.browser),
browser_version:
if(session.browser_version == "",
do: event.browser_version,
else: session.browser_version
),
screen_size:
if(session.screen_size == "", do: event.screen_size, else: session.screen_size),
events: session.events + 1
}
end
defp new_session_from_event(event) do
%Plausible.ClickhouseSession{
sign: 1,
session_id: Plausible.ClickhouseSession.random_uint64(),
hostname: event.hostname,
domain: event.domain,
user_id: event.user_id,
entry_page: event.pathname,
exit_page: event.pathname,
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),
events: 1,
referrer: event.referrer,
referrer_source: event.referrer_source,
utm_medium: event.utm_medium,
utm_source: event.utm_source,
utm_campaign: event.utm_campaign,
utm_content: event.utm_content,
utm_term: event.utm_term,
country_code: event.country_code,
subdivision1_code: event.subdivision1_code,
subdivision2_code: event.subdivision2_code,
city_geoname_id: event.city_geoname_id,
screen_size: event.screen_size,
operating_system: event.operating_system,
operating_system_version: event.operating_system_version,
browser: event.browser,
browser_version: event.browser_version,
timestamp: event.timestamp,
start: event.timestamp,
"entry_meta.key": Map.get(event, :"meta.key"),
"entry_meta.value": Map.get(event, :"meta.value")
}
end
end

@ -2,6 +2,8 @@ defmodule Plausible.ClickhouseSession do
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset import Ecto.Changeset
@type t() :: %__MODULE__{}
@primary_key false @primary_key false
schema "sessions" do schema "sessions" do
field :hostname, :string field :hostname, :string

@ -9,9 +9,9 @@ defmodule PlausibleWeb.Api.ExternalController do
require Logger require Logger
def event(conn, _params) do def event(conn, _params) do
with {:ok, ingestion_request} <- Plausible.Ingestion.Request.build(conn), with {:ok, requests} <- Plausible.Ingestion.Request.build(conn),
_ <- Sentry.Context.set_extra_context(%{request: ingestion_request}), _ <- Sentry.Context.set_extra_context(%{request: requests}),
:ok <- Plausible.Ingestion.Event.build_and_buffer(ingestion_request) do :ok <- Plausible.Ingestion.from_requests(requests) do
conn |> put_status(202) |> text("ok") conn |> put_status(202) |> text("ok")
else else
:skip -> :skip ->

@ -107,6 +107,7 @@ defmodule Plausible.MixProject do
{:opentelemetry_phoenix, "~> 1.0"}, {:opentelemetry_phoenix, "~> 1.0"},
{:telemetry, "~> 1.0", override: true}, {:telemetry, "~> 1.0", override: true},
{:opentelemetry_ecto, "~> 1.0.0"}, {:opentelemetry_ecto, "~> 1.0.0"},
{:open_telemetry_decorator, "~> 1.0.0-rc.3"},
{:observer_cli, "~> 1.7"}, {:observer_cli, "~> 1.7"},
{:mimic, "~> 1.7", only: :test}, {:mimic, "~> 1.7", only: :test},
{:prom_ex, "~> 1.7.1"}, {:prom_ex, "~> 1.7.1"},

@ -25,6 +25,7 @@
"ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
"db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"}, "db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"},
"decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
"decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"},
"dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"},
"double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"}, "double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"},
"ecto": {:hex, :ecto, "3.8.1", "35e0bd8c8eb772e14a5191a538cd079706ecb45164ea08a7523b4fc69ab70f56", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f1b68f8d5fe3ab89e24f57c03db5b5d0aed3602077972098b3a6006a1be4b69b"}, "ecto": {:hex, :ecto, "3.8.1", "35e0bd8c8eb772e14a5191a538cd079706ecb45164ea08a7523b4fc69ab70f56", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f1b68f8d5fe3ab89e24f57c03db5b5d0aed3602077972098b3a6006a1be4b69b"},
@ -76,6 +77,7 @@
"oauther": {:hex, :oauther, "1.3.0", "82b399607f0ca9d01c640438b34d74ebd9e4acd716508f868e864537ecdb1f76", [:mix], [], "hexpm", "78eb888ea875c72ca27b0864a6f550bc6ee84f2eeca37b093d3d833fbcaec04e"}, "oauther": {:hex, :oauther, "1.3.0", "82b399607f0ca9d01c640438b34d74ebd9e4acd716508f868e864537ecdb1f76", [:mix], [], "hexpm", "78eb888ea875c72ca27b0864a6f550bc6ee84f2eeca37b093d3d833fbcaec04e"},
"oban": {:hex, :oban, "2.12.0", "bd5a283770c6ab1284aad81e5566cfb89f4119b08f52508d92d73551283c8789", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1557b7b046b13c0b5360f55a9fb7e56975f6b5f8247e56f2c54575bd95435ca0"}, "oban": {:hex, :oban, "2.12.0", "bd5a283770c6ab1284aad81e5566cfb89f4119b08f52508d92d73551283c8789", [:mix], [{:ecto_sql, "~> 3.6", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1557b7b046b13c0b5360f55a9fb7e56975f6b5f8247e56f2c54575bd95435ca0"},
"observer_cli": {:hex, :observer_cli, "1.7.3", "25d094d485f47239f218b53df0691a102fef13071dfd0d04922b5142297cfc93", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "a41b6d3e11a3444e063e09cc225f7f3e631ce14019e5fbcaebfda89b1bd788ea"}, "observer_cli": {:hex, :observer_cli, "1.7.3", "25d094d485f47239f218b53df0691a102fef13071dfd0d04922b5142297cfc93", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "a41b6d3e11a3444e063e09cc225f7f3e631ce14019e5fbcaebfda89b1bd788ea"},
"open_telemetry_decorator": {:hex, :open_telemetry_decorator, "1.0.0-rc.3", "5508ee00005f7e3fc23bbf5bf527ebb48d5991691801acb8f8a815cbed985483", [:mix], [{:decorator, "~> 1.4.0", [hex: :decorator, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.0.0-rc.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "fc695170a5009737f64ef6cee75af2e26d10f9a46c3db8cbe3a02bcf0bd1f15b"},
"opentelemetry": {:hex, :opentelemetry, "1.0.2", "9ffa9ddcbec9356154681bc9d0a54bb20f0de0e8c6696ccc298b49633308782b", [:rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "e774506333c8afd9133fee4ba69d06e6413ccad6ee76f10e59c3ec624009af23"}, "opentelemetry": {:hex, :opentelemetry, "1.0.2", "9ffa9ddcbec9356154681bc9d0a54bb20f0de0e8c6696ccc298b49633308782b", [:rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "e774506333c8afd9133fee4ba69d06e6413ccad6ee76f10e59c3ec624009af23"},
"opentelemetry_api": {:hex, :opentelemetry_api, "1.0.3", "77f9644c42340cd8b18c728cde4822ed55ae136f0d07761b78e8c54da46af93a", [:mix, :rebar3], [], "hexpm", "4293e06bd369bc004e6fad5edbb56456d891f14bd3f9f1772b18f1923e0678ea"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.0.3", "77f9644c42340cd8b18c728cde4822ed55ae136f0d07761b78e8c54da46af93a", [:mix, :rebar3], [], "hexpm", "4293e06bd369bc004e6fad5edbb56456d891f14bd3f9f1772b18f1923e0678ea"},
"opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.0.0", "8463cc466429b3e125ae236a5cc32869ea38c88eb2f07722dd88e78ceca6963c", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.1.0", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "39cc60dbdd54e371bd12fb010ee960de851d782dca197efefa5c35eb6b4c18bd"}, "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.0.0", "8463cc466429b3e125ae236a5cc32869ea38c88eb2f07722dd88e78ceca6963c", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.1.0", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "39cc60dbdd54e371bd12fb010ee960de851d782dca197efefa5c35eb6b4c18bd"},

@ -1,74 +0,0 @@
defmodule Plausible.Ingestion.EventTest do
use Plausible.DataCase
def get_event(domain) do
Plausible.TestUtils.eventually(fn ->
Plausible.Event.WriteBuffer.flush()
event =
Plausible.ClickhouseRepo.one(
from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
)
{!is_nil(event), event}
end)
end
@valid_request %Plausible.Ingestion.Request{
remote_ip: "2.2.2.2",
user_agent:
"Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405",
event_name: "pageview",
url: "http://skywalker.test",
referrer: "http://m.facebook.test/",
screen_width: 1440,
hash_mode: nil,
query_params: %{
"utm_medium" => "utm_medium",
"utm_source" => "utm_source",
"utm_campaign" => "utm_campaign",
"utm_content" => "utm_content",
"utm_term" => "utm_term",
"source" => "source",
"ref" => "ref"
}
}
test "build_and_buffer/3 creates an event" do
assert :ok ==
@valid_request
|> Map.put(:domain, "plausible-ingestion-event-basic.test")
|> Plausible.Ingestion.Event.build_and_buffer()
assert %Plausible.ClickhouseEvent{
session_id: session_id,
user_id: user_id,
domain: "plausible-ingestion-event-basic.test",
browser: "Safari",
browser_version: "",
city_geoname_id: 2_988_507,
country_code: "FR",
hostname: "skywalker.test",
"meta.key": [],
"meta.value": [],
name: "pageview",
operating_system: "iOS",
operating_system_version: "3.2",
pathname: "/",
referrer: "m.facebook.test",
referrer_source: "utm_source",
screen_size: "Desktop",
subdivision1_code: "FR-IDF",
subdivision2_code: "FR-75",
transferred_from: "",
utm_campaign: "utm_campaign",
utm_content: "utm_content",
utm_medium: "utm_medium",
utm_source: "utm_source",
utm_term: "utm_term"
} = get_event("plausible-ingestion-event-basic.test")
assert is_integer(session_id)
assert is_integer(user_id)
end
end

@ -1,124 +0,0 @@
defmodule Plausible.Session.CacheStoreTest do
use Plausible.DataCase
import Double
alias Plausible.Session.{CacheStore, WriteBuffer}
setup do
buffer =
WriteBuffer
|> stub(:insert, fn _sessions -> nil end)
[buffer: buffer]
end
test "creates a session from an event", %{buffer: buffer} do
event =
build(:event,
name: "pageview",
referrer: "ref",
referrer_source: "refsource",
utm_medium: "medium",
utm_source: "source",
utm_campaign: "campaign",
utm_content: "content",
utm_term: "term",
browser: "browser",
browser_version: "55",
country_code: "EE",
screen_size: "Desktop",
operating_system: "Mac",
operating_system_version: "11",
"meta.key": ["logged_in", "darkmode"],
"meta.value": ["true", "false"]
)
CacheStore.on_event(event, nil, buffer)
assert_receive({WriteBuffer, :insert, [sessions]})
assert [session] = sessions
assert session.hostname == event.hostname
assert session.domain == event.domain
assert session.user_id == event.user_id
assert session.entry_page == event.pathname
assert session.exit_page == event.pathname
assert session.is_bounce == true
assert session.duration == 0
assert session.pageviews == 1
assert session.events == 1
assert session.referrer == event.referrer
assert session.referrer_source == event.referrer_source
assert session.utm_medium == event.utm_medium
assert session.utm_source == event.utm_source
assert session.utm_campaign == event.utm_campaign
assert session.utm_content == event.utm_content
assert session.utm_term == event.utm_term
assert session.country_code == event.country_code
assert session.screen_size == event.screen_size
assert session.operating_system == event.operating_system
assert session.operating_system_version == event.operating_system_version
assert session.browser == event.browser
assert session.browser_version == event.browser_version
assert session.timestamp == event.timestamp
assert session.start === event.timestamp
# assert Map.get(session, :"entry.meta.key") == ["logged_in", "darkmode"]
# assert Map.get(session, :"entry.meta.value") == ["true", "false"]
end
test "updates a session", %{buffer: buffer} do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: -10))
event2 =
build(:event,
domain: event1.domain,
user_id: event1.user_id,
name: "pageview",
timestamp: timestamp,
country_code: "US",
subdivision1_code: "SUB1",
subdivision2_code: "SUB2",
city_geoname_id: 12312,
screen_size: "Desktop",
operating_system: "Mac",
operating_system_version: "11",
browser: "Firefox",
browser_version: "10"
)
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)
assert_receive({WriteBuffer, :insert, [[session, _negative_record]]})
assert session.is_bounce == false
assert session.duration == 10
assert session.pageviews == 2
assert session.events == 2
assert session.country_code == "US"
assert session.subdivision1_code == "SUB1"
assert session.subdivision2_code == "SUB2"
assert session.city_geoname_id == 12312
assert session.operating_system == "Mac"
assert session.operating_system_version == "11"
assert session.browser == "Firefox"
assert session.browser_version == "10"
assert session.screen_size == "Desktop"
end
test "calculates duration correctly for out-of-order events", %{buffer: buffer} do
timestamp = Timex.now()
event1 = build(:event, name: "pageview", timestamp: timestamp |> Timex.shift(seconds: 10))
event2 =
build(:event,
domain: event1.domain,
user_id: event1.user_id,
name: "pageview",
timestamp: timestamp
)
CacheStore.on_event(event1, nil, buffer)
CacheStore.on_event(event2, nil, buffer)
assert_receive({WriteBuffer, :insert, [[session, _negative_record]]})
assert session.duration == 10
end
end

@ -3,24 +3,39 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
use Mimic use Mimic
use Plausible.ClickhouseRepo use Plausible.ClickhouseRepo
defp get_event(domain) do defp put_ip_address(conn) do
Plausible.Event.WriteBuffer.flush() ip = Enum.map_join(1..4, ".", fn _part -> :rand.uniform(512) end)
put_req_header(conn, "x-forwarded-for", ip)
end
ClickhouseRepo.one( defp get_event(domain) do
from e in Plausible.ClickhouseEvent, Plausible.TestUtils.eventually(fn ->
where: e.domain == ^domain, Plausible.Event.WriteBuffer.flush()
order_by: [desc: e.timestamp]
) event =
ClickhouseRepo.one(
from e in Plausible.ClickhouseEvent,
where: e.domain == ^domain,
order_by: [desc: e.timestamp]
)
{!is_nil(event), event}
end)
end end
defp get_events(domain) do defp get_events(domain, count) do
Plausible.Event.WriteBuffer.flush() Plausible.TestUtils.eventually(fn ->
Plausible.Event.WriteBuffer.flush()
ClickhouseRepo.all( events =
from e in Plausible.ClickhouseEvent, ClickhouseRepo.all(
where: e.domain == ^domain, from e in Plausible.ClickhouseEvent,
order_by: [desc: e.timestamp] where: e.domain == ^domain,
) order_by: [desc: e.timestamp]
)
{length(events) == count, events}
end)
end end
@user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
@ -37,7 +52,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("x-forwarded-for", "2.2.2.2")
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-1.com") pageview = get_event("external-controller-test-1.com")
@ -59,6 +74,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("content-type", "text/plain") |> put_req_header("content-type", "text/plain")
|> put_ip_address()
|> post("/api/event", Jason.encode!(params)) |> post("/api/event", Jason.encode!(params))
pageview = get_event("external-controller-test-text-plain.com") pageview = get_event("external-controller-test-text-plain.com")
@ -73,6 +89,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("content-type", "text/plain") |> put_req_header("content-type", "text/plain")
|> put_ip_address()
|> post("/api/event", "") |> post("/api/event", "")
assert conn.status == 400 assert conn.status == 400
@ -90,6 +107,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert response(conn, 202) == "ok" assert response(conn, 202) == "ok"
@ -105,6 +123,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-2.com") pageview = get_event("external-controller-test-2.com")
@ -120,6 +139,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-3.com") pageview = get_event("external-controller-test-3.com")
@ -135,6 +155,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-4.com") pageview = get_event("external-controller-test-4.com")
@ -150,6 +171,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-trailing-whitespace.com") pageview = get_event("external-controller-test-trailing-whitespace.com")
@ -166,6 +188,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn conn
|> put_req_header("user-agent", "generic crawler") |> put_req_header("user-agent", "generic crawler")
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert get_event("external-controller-test-5.com") == nil assert get_event("external-controller-test-5.com") == nil
@ -183,6 +206,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
"user-agent", "user-agent",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/85.0.4183.83 Safari/537.36" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/85.0.4183.83 Safari/537.36"
) )
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert get_event("headless-chrome-test.com") == nil assert get_event("headless-chrome-test.com") == nil
@ -198,6 +222,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-6.com") pageview = get_event("external-controller-test-6.com")
@ -220,6 +245,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-7.com") pageview = get_event("external-controller-test-7.com")
@ -239,6 +265,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-8.com") pageview = get_event("external-controller-test-8.com")
@ -260,6 +287,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert response(conn, 202) == "ok" assert response(conn, 202) == "ok"
@ -269,8 +297,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "feature flag - blocks traffic from a domain when block_traffic is enabled", %{ test "feature flag - blocks traffic from a domain when block_traffic is enabled", %{
conn: conn conn: conn
} do } do
FunWithFlags set_mimic_global()
|> stub(:enabled?, fn
stub(FunWithFlags, :enabled?, fn
:block_event_ingest, [for: "feature-flag-test.com"] -> true :block_event_ingest, [for: "feature-flag-test.com"] -> true
_, _ -> false _, _ -> false
end) end)
@ -284,6 +313,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert response(conn, 202) == "ok" assert response(conn, 202) == "ok"
@ -301,6 +331,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-9.com") pageview = get_event("external-controller-test-9.com")
@ -320,6 +351,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-10.com") pageview = get_event("external-controller-test-10.com")
@ -339,6 +371,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-11.com") pageview = get_event("external-controller-test-11.com")
@ -356,6 +389,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-12.com") pageview = get_event("external-controller-test-12.com")
@ -372,6 +406,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-13.com") pageview = get_event("external-controller-test-13.com")
@ -388,6 +423,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-utm-tags.com") pageview = get_event("external-controller-test-utm-tags.com")
@ -408,6 +444,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-14.com") pageview = get_event("external-controller-test-14.com")
@ -427,6 +464,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-15.com") pageview = get_event("external-controller-test-15.com")
@ -446,6 +484,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-16.com") pageview = get_event("external-controller-test-16.com")
@ -464,6 +503,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-17.com") pageview = get_event("external-controller-test-17.com")
@ -482,6 +522,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
event = get_event("external-controller-test-18.com") event = get_event("external-controller-test-18.com")
@ -502,6 +543,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
event = get_event("custom-prop-test.com") event = get_event("custom-prop-test.com")
@ -519,6 +561,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
event = get_event("custom-prop-test-2.com") event = get_event("custom-prop-test-2.com")
@ -536,6 +579,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
event = get_event("custom-prop-test-3.com") event = get_event("custom-prop-test-3.com")
@ -552,7 +596,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
props: Jason.encode!(%{wat: ["some-thing"], other: "key"}) props: Jason.encode!(%{wat: ["some-thing"], other: "key"})
} }
conn = post(conn, "/api/event", params) conn =
conn
|> put_ip_address()
|> post("/api/event", params)
assert conn.status == 202 assert conn.status == 202
@ -570,7 +617,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
props: Jason.encode!(%{foo: %{bar: "baz"}, other_key: 1}) props: Jason.encode!(%{foo: %{bar: "baz"}, other_key: 1})
} }
conn = post(conn, "/api/event", params) conn =
conn
|> put_ip_address()
|> post("/api/event", params)
assert conn.status == 202 assert conn.status == 202
@ -588,7 +638,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
props: Jason.encode!(%{foo: "", other_key: true}) props: Jason.encode!(%{foo: "", other_key: true})
} }
conn = post(conn, "/api/event", params) conn =
conn
|> put_ip_address()
|> post("/api/event", params)
assert conn.status == 202 assert conn.status == 202
@ -606,7 +659,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
props: Jason.encode!(%{foo: nil, other_key: true}) props: Jason.encode!(%{foo: nil, other_key: true})
} }
conn = post(conn, "/api/event", params) conn =
conn
|> put_ip_address()
|> post("/api/event", params)
assert conn.status == 202 assert conn.status == 202
@ -627,6 +683,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
event = get_event("external-controller-test-19.com") event = get_event("external-controller-test-19.com")
@ -799,6 +856,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-21.com") pageview = get_event("external-controller-test-21.com")
@ -816,6 +874,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-22.com") pageview = get_event("external-controller-test-22.com")
@ -835,6 +894,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-23.com") pageview = get_event("external-controller-test-23.com")
@ -851,6 +911,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("external-controller-test-hash-0.com") pageview = get_event("external-controller-test-hash-0.com")
@ -868,6 +929,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("url-decode-test.com") pageview = get_event("url-decode-test.com")
@ -888,6 +950,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
} }
conn conn
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
pageview = get_event("quote-encode-test.com") pageview = get_event("quote-encode-test.com")
@ -895,7 +958,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageview.utm_source == "Something \"quoted\"" assert pageview.utm_source == "Something \"quoted\""
end end
test "responds 400 when required fields are missing", %{conn: conn} do test "responds 202 when required fields are missing", %{conn: conn} do
params = %{ params = %{
domain: "some-domain.com", domain: "some-domain.com",
name: "pageview" name: "pageview"
@ -904,16 +967,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert json_response(conn, 400) == %{ # TODO: Test events with validation errors are not persisted in
"errors" => %{ # Plausible.Ingestion.UserTest
"hostname" => ["can't be blank"] assert response(conn, 202) == "ok"
}
}
end end
test "responds 400 with errors when domain is missing", %{conn: conn} do test "responds 202 when domain is missing", %{conn: conn} do
params = %{ params = %{
domain: nil, domain: nil,
url: "about:config", url: "about:config",
@ -923,15 +985,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("user-agent", @user_agent) |> put_req_header("user-agent", @user_agent)
|> put_ip_address()
|> post("/api/event", params) |> post("/api/event", params)
assert json_response(conn, 400) == %{ # TODO: Test events with validation errors are not persisted in
"errors" => %{ # Plausible.Ingestion.Test
"domain" => ["can't be blank"], assert response(conn, 202) == "ok"
"hostname" => ["can't be blank"],
"user_id" => ["can't be blank"]
}
}
end end
end end
@ -953,7 +1012,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1") |> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", params) |> post("/api/event", params)
[one, two] = get_events("user-id-test-domain.com") [one, two] = get_events("user-id-test-domain.com", 2)
assert one.user_id == two.user_id assert one.user_id == two.user_id
end end
@ -975,7 +1034,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "982.32.12.1") |> put_req_header("x-forwarded-for", "982.32.12.1")
|> post("/api/event", params) |> post("/api/event", params)
[one, two] = get_events("user-id-test-domain-2.com") [one, two] = get_events("user-id-test-domain-2.com", 2)
assert one.user_id != two.user_id assert one.user_id != two.user_id
end end
@ -997,7 +1056,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1") |> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", params) |> post("/api/event", params)
[one, two] = get_events("user-id-test-domain-3.com") [one, two] = get_events("user-id-test-domain-3.com", 2)
assert one.user_id != two.user_id assert one.user_id != two.user_id
end end
@ -1042,7 +1101,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1") |> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", Map.put(params, :url, "https://other-domain.com/")) |> post("/api/event", Map.put(params, :url, "https://other-domain.com/"))
[one, two] = get_events("user-id-test-domain-5.com") [one, two] = get_events("user-id-test-domain-5.com", 2)
assert one.user_id != two.user_id assert one.user_id != two.user_id
end end
@ -1066,7 +1125,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> put_req_header("x-forwarded-for", "127.0.0.1") |> put_req_header("x-forwarded-for", "127.0.0.1")
|> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/")) |> post("/api/event", Map.put(params, :url, "https://app.user-id-test-domain.com/"))
[one, two] = get_events("user-id-test-domain-6.com") [one, two] = get_events("user-id-test-domain-6.com", 2)
assert one.user_id == two.user_id assert one.user_id == two.user_id
end end
@ -1082,6 +1141,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("content-type", "text/plain") |> put_req_header("content-type", "text/plain")
|> put_ip_address()
|> post("/api/event", Jason.encode!(params)) |> post("/api/event", Jason.encode!(params))
pageview = get_event("url-with-hostname-missing.com") pageview = get_event("url-with-hostname-missing.com")
@ -1100,6 +1160,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
conn = conn =
conn conn
|> put_req_header("content-type", "text/plain") |> put_req_header("content-type", "text/plain")
|> put_ip_address()
|> post("/api/event", Jason.encode!(params)) |> post("/api/event", Jason.encode!(params))
pageview = get_event("chrome-extension-url.com") pageview = get_event("chrome-extension-url.com")

@ -186,12 +186,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
domain: site.domain, domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00] timestamp: ~N[2021-01-01 00:00:00]
), ),
build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])
])
populate_stats([
build(:pageview, build(:pageview,
user_id: @user_id, user_id: @user_id,
domain: site.domain, domain: site.domain,
timestamp: ~N[2021-02-01 00:25:00] timestamp: ~N[2021-02-01 00:25:00]
), )
build(:pageview, domain: site.domain, timestamp: ~N[2021-03-01 00:00:00])
]) ])
conn = conn =

@ -627,6 +627,14 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
end end
test "returns top entry pages by visitors with imported data", %{conn: conn, site: site} do test "returns top entry pages by visitors with imported data", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/page2",
user_id: @user_id,
timestamp: ~N[2021-01-01 23:15:00]
)
])
populate_stats(site, [ populate_stats(site, [
build(:pageview, build(:pageview,
pathname: "/page1", pathname: "/page1",
@ -645,11 +653,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
pathname: "/page2", pathname: "/page2",
user_id: @user_id, user_id: @user_id,
timestamp: ~N[2021-01-01 00:15:00] timestamp: ~N[2021-01-01 00:15:00]
),
build(:pageview,
pathname: "/page2",
user_id: @user_id,
timestamp: ~N[2021-01-01 23:15:00]
) )
]) ])

@ -113,21 +113,22 @@ defmodule Plausible.TestUtils do
end end
end) end)
if native, do: populate_native_stats(native) populate_native_stats(native)
if imported, do: populate_imported_stats(imported) populate_imported_stats(imported)
end end
defp populate_native_stats(events) do defp populate_native_stats(events) do
sessions = events
Enum.reduce(events, %{}, fn event, sessions -> |> Enum.map(fn event ->
session_id = Plausible.Session.CacheStore.on_event(event, nil) pid =
Map.put(sessions, {event.domain, event.user_id}, session_id) Plausible.Ingestion.Session.DynamicSupervisor.find_or_spawn(event.domain, event.user_id)
end)
:ok = Plausible.Ingestion.Session.Actor.send_event(pid, event)
Enum.each(events, fn event -> pid
event = Map.put(event, :session_id, sessions[{event.domain, event.user_id}])
Plausible.Event.WriteBuffer.insert(event)
end) end)
|> Enum.uniq()
|> Enum.each(&GenServer.stop(&1))
Plausible.Session.WriteBuffer.flush() Plausible.Session.WriteBuffer.flush()
Plausible.Event.WriteBuffer.flush() Plausible.Event.WriteBuffer.flush()

@ -1,6 +1,6 @@
{:ok, _} = Application.ensure_all_started(:ex_machina) {:ok, _} = Application.ensure_all_started(:ex_machina)
Plausible.Test.ClickhouseSetup.run() Plausible.Test.ClickhouseSetup.run()
Mimic.copy(FunWithFlags) Mimic.copy(FunWithFlags)
ExUnit.start() ExUnit.start(capture_log: true)
Application.ensure_all_started(:double) Application.ensure_all_started(:double)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual) Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)

Loading…
Cancel
Save