Revert "experiment in improving throughput of write buffers (#3511)" (#3614)

This reverts commit 16bfb11701.
pull/3615/head r2023.12.11-280
hq1 3 years ago committed by GitHub
parent 69579272d6
commit 7b5c20db17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -27,7 +27,6 @@ All notable changes to this project will be documented in this file.
- Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal
- Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard
- GA/SC sections moved to new settings: Integrations
- `CLICKHOUSE_MAX_BUFFER_SIZE` now represents bytes and defaults to `100000` (100KB)
### Fixed
- Only return `(none)` values in custom property breakdown for the first page (pagination) of results

@ -142,7 +142,7 @@ ch_db_url =
{ch_max_buffer_size, ""} =
config_dir
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "100000")
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
|> Integer.parse()
# Can be generated with `Base.encode64(:crypto.strong_rand_bytes(32))` from

@ -17,8 +17,8 @@ defmodule Plausible.Application do
{Finch, name: Plausible.Finch, pools: finch_pool_config()},
{Phoenix.PubSub, name: Plausible.PubSub},
Plausible.Session.Salts,
Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),
Plausible.Event.WriteBuffer,
Plausible.Session.WriteBuffer,
ReferrerBlocklist,
Supervisor.child_spec({Cachex, name: :user_agents, limit: 10_000, stats: true},
id: :cachex_user_agents

@ -1,38 +1,79 @@
defmodule Plausible.Event.WriteBuffer do
@moduledoc false
use GenServer
require Logger
%{
header: header,
insert_sql: insert_sql,
insert_opts: insert_opts,
fields: fields,
encoding_types: encoding_types
} =
Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2)
alias Plausible.IngestRepo
def child_spec(opts) do
opts =
Keyword.merge(opts,
name: __MODULE__,
header: unquote(header),
insert_sql: unquote(insert_sql),
insert_opts: unquote(insert_opts)
)
def start_link(_opts) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
Plausible.Ingestion.WriteBuffer.child_spec(opts)
def init(buffer) do
Process.flag(:trap_exit, true)
timer = Process.send_after(self(), :tick, flush_interval_ms())
{:ok, %{buffer: buffer, timer: timer}}
end
def insert(event) do
row_binary =
[Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)]
|> Ch.RowBinary._encode_rows(unquote(encoding_types))
|> IO.iodata_to_binary()
:ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)
GenServer.cast(__MODULE__, {:insert, event})
{:ok, event}
end
def flush do
Plausible.Ingestion.WriteBuffer.flush(__MODULE__)
def flush() do
GenServer.call(__MODULE__, :flush, :infinity)
:ok
end
def handle_cast({:insert, event}, %{buffer: buffer} = state) do
new_buffer = [event | buffer]
if length(new_buffer) >= max_buffer_size() do
Logger.info("Buffer full, flushing to disk")
Process.cancel_timer(state[:timer])
do_flush(new_buffer)
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
{:noreply, %{buffer: [], timer: new_timer}}
else
{:noreply, %{state | buffer: new_buffer}}
end
end
def handle_info(:tick, %{buffer: buffer}) do
do_flush(buffer)
timer = Process.send_after(self(), :tick, flush_interval_ms())
{:noreply, %{buffer: [], timer: timer}}
end
def handle_call(:flush, _from, %{buffer: buffer} = state) do
Process.cancel_timer(state[:timer])
do_flush(buffer)
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
{:reply, nil, %{buffer: [], timer: new_timer}}
end
def terminate(_reason, %{buffer: buffer}) do
Logger.info("Flushing event buffer before shutdown...")
do_flush(buffer)
end
defp do_flush(buffer) do
case buffer do
[] ->
nil
events ->
Logger.info("Flushing #{length(events)} events")
events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))
IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
end
end
defp flush_interval_ms() do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
end
defp max_buffer_size() do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
end
end

@ -1,150 +0,0 @@
defmodule Plausible.Ingestion.WriteBuffer do
@moduledoc false
use GenServer
require Logger
alias Plausible.IngestRepo
def start_link(opts) do
GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name))
end
def insert(server, row_binary) do
GenServer.cast(server, {:insert, row_binary})
end
def flush(server) do
GenServer.call(server, :flush, :infinity)
end
@impl true
def init(opts) do
buffer = opts[:buffer] || []
max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size()
flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms()
Process.flag(:trap_exit, true)
timer = Process.send_after(self(), :tick, flush_interval_ms)
{:ok,
%{
buffer: buffer,
timer: timer,
name: Keyword.fetch!(opts, :name),
insert_sql: Keyword.fetch!(opts, :insert_sql),
insert_opts: Keyword.fetch!(opts, :insert_opts),
header: Keyword.fetch!(opts, :header),
buffer_size: IO.iodata_length(buffer),
max_buffer_size: max_buffer_size,
flush_interval_ms: flush_interval_ms
}}
end
@impl true
def handle_cast({:insert, row_binary}, state) do
state = %{
state
| buffer: [state.buffer | row_binary],
buffer_size: state.buffer_size + IO.iodata_length(row_binary)
}
if state.buffer_size >= state.max_buffer_size do
Logger.info("#{state.name} buffer full, flushing to ClickHouse")
Process.cancel_timer(state.timer)
do_flush(state)
new_timer = Process.send_after(self(), :tick, state.flush_interval_ms)
{:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}}
else
{:noreply, state}
end
end
@impl true
def handle_info(:tick, state) do
do_flush(state)
timer = Process.send_after(self(), :tick, state.flush_interval_ms)
{:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}}
end
@impl true
def handle_call(:flush, _from, state) do
%{timer: timer, flush_interval_ms: flush_interval_ms} = state
Process.cancel_timer(timer)
do_flush(state)
new_timer = Process.send_after(self(), :tick, flush_interval_ms)
{:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}}
end
@impl true
def terminate(_reason, %{name: name} = state) do
Logger.info("Flushing #{name} buffer before shutdown...")
do_flush(state)
end
defp do_flush(state) do
%{
buffer: buffer,
buffer_size: buffer_size,
insert_opts: insert_opts,
insert_sql: insert_sql,
header: header,
name: name
} = state
case buffer do
[] ->
nil
_not_empty ->
Logger.info("Flushing #{buffer_size} byte(s) RowBinary from #{name}")
IngestRepo.query!(insert_sql, [header | buffer], insert_opts)
end
end
defp default_flush_interval_ms do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
end
defp default_max_buffer_size do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
end
@doc false
def compile_time_prepare(schema) do
fields = schema.__schema__(:fields)
types =
Enum.map(fields, fn field ->
type = schema.__schema__(:type, field) || raise "missing type for #{field}"
type
|> Ecto.Type.type()
|> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field)
end)
encoding_types = Ch.RowBinary.encoding_types(types)
header =
fields
|> Enum.map(&to_string/1)
|> Ch.RowBinary.encode_names_and_types(types)
|> IO.iodata_to_binary()
insert_sql = "INSERT INTO #{schema.__schema__(:source)} FORMAT RowBinaryWithNamesAndTypes"
%{
fields: fields,
types: types,
encoding_types: encoding_types,
header: header,
insert_sql: insert_sql,
insert_opts: [
command: :insert,
encode: false,
source: schema.__schema__(:source),
cast_params: []
]
}
end
end

@ -1,43 +1,83 @@
defmodule Plausible.Session.WriteBuffer do
@moduledoc false
use GenServer
require Logger
%{
header: header,
insert_sql: insert_sql,
insert_opts: insert_opts,
fields: fields,
encoding_types: encoding_types
} =
Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseSessionV2)
alias Plausible.IngestRepo
def child_spec(opts) do
opts =
Keyword.merge(opts,
name: __MODULE__,
header: unquote(header),
insert_sql: unquote(insert_sql),
insert_opts: unquote(insert_opts)
)
def start_link(_opts) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
Plausible.Ingestion.WriteBuffer.child_spec(opts)
def init(buffer) do
Process.flag(:trap_exit, true)
timer = Process.send_after(self(), :tick, flush_interval_ms())
{:ok, %{buffer: buffer, timer: timer}}
end
def insert(sessions) do
row_binary =
sessions
|> Enum.map(fn %{is_bounce: is_bounce} = session ->
{:ok, is_bounce} = Plausible.ClickhouseSessionV2.BoolUInt8.dump(is_bounce)
session = %{session | is_bounce: is_bounce}
Enum.map(unquote(fields), fn field -> Map.fetch!(session, field) end)
end)
|> Ch.RowBinary._encode_rows(unquote(encoding_types))
|> IO.iodata_to_binary()
:ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)
GenServer.cast(__MODULE__, {:insert, sessions})
{:ok, sessions}
end
def flush do
Plausible.Ingestion.WriteBuffer.flush(__MODULE__)
def flush() do
GenServer.call(__MODULE__, :flush, :infinity)
:ok
end
def handle_cast({:insert, sessions}, %{buffer: buffer} = state) do
new_buffer = sessions ++ buffer
if length(new_buffer) >= max_buffer_size() do
Logger.info("Buffer full, flushing to disk")
Process.cancel_timer(state[:timer])
do_flush(new_buffer)
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
{:noreply, %{buffer: [], timer: new_timer}}
else
{:noreply, %{state | buffer: new_buffer}}
end
end
def handle_info(:tick, %{buffer: buffer}) do
do_flush(buffer)
timer = Process.send_after(self(), :tick, flush_interval_ms())
{:noreply, %{buffer: [], timer: timer}}
end
def handle_call(:flush, _from, %{buffer: buffer} = state) do
Process.cancel_timer(state[:timer])
do_flush(buffer)
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
{:reply, nil, %{buffer: [], timer: new_timer}}
end
def terminate(_reason, %{buffer: buffer}) do
Logger.info("Flushing session buffer before shutdown...")
do_flush(buffer)
end
defp do_flush(buffer) do
case buffer do
[] ->
nil
sessions ->
Logger.info("Flushing #{length(sessions)} sessions")
sessions =
sessions
|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))
|> Enum.reverse()
IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
end
end
defp flush_interval_ms() do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
end
defp max_buffer_size() do
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
end
end

Loading…
Cancel
Save