This reverts commit 16bfb11701.
pull/3615/head
r2023.12.11-280
parent
69579272d6
commit
7b5c20db17
@ -1,38 +1,79 @@
|
||||
defmodule Plausible.Event.WriteBuffer do
|
||||
@moduledoc false
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
%{
|
||||
header: header,
|
||||
insert_sql: insert_sql,
|
||||
insert_opts: insert_opts,
|
||||
fields: fields,
|
||||
encoding_types: encoding_types
|
||||
} =
|
||||
Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2)
|
||||
alias Plausible.IngestRepo
|
||||
|
||||
def child_spec(opts) do
|
||||
opts =
|
||||
Keyword.merge(opts,
|
||||
name: __MODULE__,
|
||||
header: unquote(header),
|
||||
insert_sql: unquote(insert_sql),
|
||||
insert_opts: unquote(insert_opts)
|
||||
)
|
||||
def start_link(_opts) do
|
||||
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
||||
end
|
||||
|
||||
Plausible.Ingestion.WriteBuffer.child_spec(opts)
|
||||
def init(buffer) do
|
||||
Process.flag(:trap_exit, true)
|
||||
timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:ok, %{buffer: buffer, timer: timer}}
|
||||
end
|
||||
|
||||
def insert(event) do
|
||||
row_binary =
|
||||
[Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)]
|
||||
|> Ch.RowBinary._encode_rows(unquote(encoding_types))
|
||||
|> IO.iodata_to_binary()
|
||||
|
||||
:ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)
|
||||
GenServer.cast(__MODULE__, {:insert, event})
|
||||
{:ok, event}
|
||||
end
|
||||
|
||||
def flush do
|
||||
Plausible.Ingestion.WriteBuffer.flush(__MODULE__)
|
||||
def flush() do
|
||||
GenServer.call(__MODULE__, :flush, :infinity)
|
||||
:ok
|
||||
end
|
||||
|
||||
def handle_cast({:insert, event}, %{buffer: buffer} = state) do
|
||||
new_buffer = [event | buffer]
|
||||
|
||||
if length(new_buffer) >= max_buffer_size() do
|
||||
Logger.info("Buffer full, flushing to disk")
|
||||
Process.cancel_timer(state[:timer])
|
||||
do_flush(new_buffer)
|
||||
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:noreply, %{buffer: [], timer: new_timer}}
|
||||
else
|
||||
{:noreply, %{state | buffer: new_buffer}}
|
||||
end
|
||||
end
|
||||
|
||||
def handle_info(:tick, %{buffer: buffer}) do
|
||||
do_flush(buffer)
|
||||
timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:noreply, %{buffer: [], timer: timer}}
|
||||
end
|
||||
|
||||
def handle_call(:flush, _from, %{buffer: buffer} = state) do
|
||||
Process.cancel_timer(state[:timer])
|
||||
do_flush(buffer)
|
||||
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:reply, nil, %{buffer: [], timer: new_timer}}
|
||||
end
|
||||
|
||||
def terminate(_reason, %{buffer: buffer}) do
|
||||
Logger.info("Flushing event buffer before shutdown...")
|
||||
do_flush(buffer)
|
||||
end
|
||||
|
||||
defp do_flush(buffer) do
|
||||
case buffer do
|
||||
[] ->
|
||||
nil
|
||||
|
||||
events ->
|
||||
Logger.info("Flushing #{length(events)} events")
|
||||
events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__)))
|
||||
|
||||
IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
|
||||
end
|
||||
end
|
||||
|
||||
defp flush_interval_ms() do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
|
||||
end
|
||||
|
||||
defp max_buffer_size() do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
|
||||
end
|
||||
end
|
||||
|
||||
@ -1,150 +0,0 @@
|
||||
defmodule Plausible.Ingestion.WriteBuffer do
|
||||
@moduledoc false
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
alias Plausible.IngestRepo
|
||||
|
||||
def start_link(opts) do
|
||||
GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name))
|
||||
end
|
||||
|
||||
def insert(server, row_binary) do
|
||||
GenServer.cast(server, {:insert, row_binary})
|
||||
end
|
||||
|
||||
def flush(server) do
|
||||
GenServer.call(server, :flush, :infinity)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(opts) do
|
||||
buffer = opts[:buffer] || []
|
||||
max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size()
|
||||
flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms()
|
||||
|
||||
Process.flag(:trap_exit, true)
|
||||
timer = Process.send_after(self(), :tick, flush_interval_ms)
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
buffer: buffer,
|
||||
timer: timer,
|
||||
name: Keyword.fetch!(opts, :name),
|
||||
insert_sql: Keyword.fetch!(opts, :insert_sql),
|
||||
insert_opts: Keyword.fetch!(opts, :insert_opts),
|
||||
header: Keyword.fetch!(opts, :header),
|
||||
buffer_size: IO.iodata_length(buffer),
|
||||
max_buffer_size: max_buffer_size,
|
||||
flush_interval_ms: flush_interval_ms
|
||||
}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
|
||||
def handle_cast({:insert, row_binary}, state) do
|
||||
state = %{
|
||||
state
|
||||
| buffer: [state.buffer | row_binary],
|
||||
buffer_size: state.buffer_size + IO.iodata_length(row_binary)
|
||||
}
|
||||
|
||||
if state.buffer_size >= state.max_buffer_size do
|
||||
Logger.info("#{state.name} buffer full, flushing to ClickHouse")
|
||||
Process.cancel_timer(state.timer)
|
||||
do_flush(state)
|
||||
new_timer = Process.send_after(self(), :tick, state.flush_interval_ms)
|
||||
{:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}}
|
||||
else
|
||||
{:noreply, state}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:tick, state) do
|
||||
do_flush(state)
|
||||
timer = Process.send_after(self(), :tick, state.flush_interval_ms)
|
||||
{:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:flush, _from, state) do
|
||||
%{timer: timer, flush_interval_ms: flush_interval_ms} = state
|
||||
Process.cancel_timer(timer)
|
||||
do_flush(state)
|
||||
new_timer = Process.send_after(self(), :tick, flush_interval_ms)
|
||||
{:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def terminate(_reason, %{name: name} = state) do
|
||||
Logger.info("Flushing #{name} buffer before shutdown...")
|
||||
do_flush(state)
|
||||
end
|
||||
|
||||
defp do_flush(state) do
|
||||
%{
|
||||
buffer: buffer,
|
||||
buffer_size: buffer_size,
|
||||
insert_opts: insert_opts,
|
||||
insert_sql: insert_sql,
|
||||
header: header,
|
||||
name: name
|
||||
} = state
|
||||
|
||||
case buffer do
|
||||
[] ->
|
||||
nil
|
||||
|
||||
_not_empty ->
|
||||
Logger.info("Flushing #{buffer_size} byte(s) RowBinary from #{name}")
|
||||
IngestRepo.query!(insert_sql, [header | buffer], insert_opts)
|
||||
end
|
||||
end
|
||||
|
||||
defp default_flush_interval_ms do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
|
||||
end
|
||||
|
||||
defp default_max_buffer_size do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def compile_time_prepare(schema) do
|
||||
fields = schema.__schema__(:fields)
|
||||
|
||||
types =
|
||||
Enum.map(fields, fn field ->
|
||||
type = schema.__schema__(:type, field) || raise "missing type for #{field}"
|
||||
|
||||
type
|
||||
|> Ecto.Type.type()
|
||||
|> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field)
|
||||
end)
|
||||
|
||||
encoding_types = Ch.RowBinary.encoding_types(types)
|
||||
|
||||
header =
|
||||
fields
|
||||
|> Enum.map(&to_string/1)
|
||||
|> Ch.RowBinary.encode_names_and_types(types)
|
||||
|> IO.iodata_to_binary()
|
||||
|
||||
insert_sql = "INSERT INTO #{schema.__schema__(:source)} FORMAT RowBinaryWithNamesAndTypes"
|
||||
|
||||
%{
|
||||
fields: fields,
|
||||
types: types,
|
||||
encoding_types: encoding_types,
|
||||
header: header,
|
||||
insert_sql: insert_sql,
|
||||
insert_opts: [
|
||||
command: :insert,
|
||||
encode: false,
|
||||
source: schema.__schema__(:source),
|
||||
cast_params: []
|
||||
]
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -1,43 +1,83 @@
|
||||
defmodule Plausible.Session.WriteBuffer do
|
||||
@moduledoc false
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
%{
|
||||
header: header,
|
||||
insert_sql: insert_sql,
|
||||
insert_opts: insert_opts,
|
||||
fields: fields,
|
||||
encoding_types: encoding_types
|
||||
} =
|
||||
Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseSessionV2)
|
||||
alias Plausible.IngestRepo
|
||||
|
||||
def child_spec(opts) do
|
||||
opts =
|
||||
Keyword.merge(opts,
|
||||
name: __MODULE__,
|
||||
header: unquote(header),
|
||||
insert_sql: unquote(insert_sql),
|
||||
insert_opts: unquote(insert_opts)
|
||||
)
|
||||
def start_link(_opts) do
|
||||
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
||||
end
|
||||
|
||||
Plausible.Ingestion.WriteBuffer.child_spec(opts)
|
||||
def init(buffer) do
|
||||
Process.flag(:trap_exit, true)
|
||||
timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:ok, %{buffer: buffer, timer: timer}}
|
||||
end
|
||||
|
||||
def insert(sessions) do
|
||||
row_binary =
|
||||
sessions
|
||||
|> Enum.map(fn %{is_bounce: is_bounce} = session ->
|
||||
{:ok, is_bounce} = Plausible.ClickhouseSessionV2.BoolUInt8.dump(is_bounce)
|
||||
session = %{session | is_bounce: is_bounce}
|
||||
Enum.map(unquote(fields), fn field -> Map.fetch!(session, field) end)
|
||||
end)
|
||||
|> Ch.RowBinary._encode_rows(unquote(encoding_types))
|
||||
|> IO.iodata_to_binary()
|
||||
|
||||
:ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)
|
||||
GenServer.cast(__MODULE__, {:insert, sessions})
|
||||
{:ok, sessions}
|
||||
end
|
||||
|
||||
def flush do
|
||||
Plausible.Ingestion.WriteBuffer.flush(__MODULE__)
|
||||
def flush() do
|
||||
GenServer.call(__MODULE__, :flush, :infinity)
|
||||
:ok
|
||||
end
|
||||
|
||||
def handle_cast({:insert, sessions}, %{buffer: buffer} = state) do
|
||||
new_buffer = sessions ++ buffer
|
||||
|
||||
if length(new_buffer) >= max_buffer_size() do
|
||||
Logger.info("Buffer full, flushing to disk")
|
||||
Process.cancel_timer(state[:timer])
|
||||
do_flush(new_buffer)
|
||||
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:noreply, %{buffer: [], timer: new_timer}}
|
||||
else
|
||||
{:noreply, %{state | buffer: new_buffer}}
|
||||
end
|
||||
end
|
||||
|
||||
def handle_info(:tick, %{buffer: buffer}) do
|
||||
do_flush(buffer)
|
||||
timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:noreply, %{buffer: [], timer: timer}}
|
||||
end
|
||||
|
||||
def handle_call(:flush, _from, %{buffer: buffer} = state) do
|
||||
Process.cancel_timer(state[:timer])
|
||||
do_flush(buffer)
|
||||
new_timer = Process.send_after(self(), :tick, flush_interval_ms())
|
||||
{:reply, nil, %{buffer: [], timer: new_timer}}
|
||||
end
|
||||
|
||||
def terminate(_reason, %{buffer: buffer}) do
|
||||
Logger.info("Flushing session buffer before shutdown...")
|
||||
do_flush(buffer)
|
||||
end
|
||||
|
||||
defp do_flush(buffer) do
|
||||
case buffer do
|
||||
[] ->
|
||||
nil
|
||||
|
||||
sessions ->
|
||||
Logger.info("Flushing #{length(sessions)} sessions")
|
||||
|
||||
sessions =
|
||||
sessions
|
||||
|> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__)))
|
||||
|> Enum.reverse()
|
||||
|
||||
IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
|
||||
end
|
||||
end
|
||||
|
||||
defp flush_interval_ms() do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms)
|
||||
end
|
||||
|
||||
defp max_buffer_size() do
|
||||
Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size)
|
||||
end
|
||||
end
|
||||
|
||||
Loading…
Reference in New Issue