diff --git a/CHANGELOG.md b/CHANGELOG.md index a48e8fd44..ad7e43877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ All notable changes to this project will be documented in this file. - Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal - Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard - GA/SC sections moved to new settings: Integrations -- `CLICKHOUSE_MAX_BUFFER_SIZE` now represents bytes and defaults to `100000` (100KB) ### Fixed - Only return `(none)` values in custom property breakdown for the first page (pagination) of results diff --git a/config/runtime.exs b/config/runtime.exs index 986a8b09d..27f1b5413 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -142,7 +142,7 @@ ch_db_url = {ch_max_buffer_size, ""} = config_dir - |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "100000") + |> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000") |> Integer.parse() # Can be generated with `Base.encode64(:crypto.strong_rand_bytes(32))` from diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 886993d12..b9d58c3e5 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -17,8 +17,8 @@ defmodule Plausible.Application do {Finch, name: Plausible.Finch, pools: finch_pool_config()}, {Phoenix.PubSub, name: Plausible.PubSub}, Plausible.Session.Salts, - Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), - Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), + Plausible.Event.WriteBuffer, + Plausible.Session.WriteBuffer, ReferrerBlocklist, Supervisor.child_spec({Cachex, name: :user_agents, limit: 10_000, stats: true}, id: :cachex_user_agents diff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.ex index 3966b5a3f..37a9cd39b 100644 --- a/lib/plausible/event/write_buffer.ex +++ b/lib/plausible/event/write_buffer.ex @@ -1,38 +1,79 @@ defmodule Plausible.Event.WriteBuffer do - @moduledoc false + use GenServer + require Logger - %{ - header: header, - insert_sql: insert_sql, - insert_opts: insert_opts, - fields: fields, - encoding_types: encoding_types - } = - Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2) + alias Plausible.IngestRepo - def child_spec(opts) do - opts = - Keyword.merge(opts, - name: __MODULE__, - header: unquote(header), - insert_sql: unquote(insert_sql), - insert_opts: unquote(insert_opts) - ) + def start_link(_opts) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end - Plausible.Ingestion.WriteBuffer.child_spec(opts) + def init(buffer) do + Process.flag(:trap_exit, true) + timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:ok, %{buffer: buffer, timer: timer}} end def insert(event) do - row_binary = - [Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)] - |> Ch.RowBinary._encode_rows(unquote(encoding_types)) - |> IO.iodata_to_binary() - - :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary) + GenServer.cast(__MODULE__, {:insert, event}) {:ok, event} end - def flush do - Plausible.Ingestion.WriteBuffer.flush(__MODULE__) + def flush() do + GenServer.call(__MODULE__, :flush, :infinity) + :ok + end + + def handle_cast({:insert, event}, %{buffer: buffer} = state) do + new_buffer = [event | buffer] + + if length(new_buffer) >= max_buffer_size() do + Logger.info("Buffer full, flushing to disk") + Process.cancel_timer(state[:timer]) + do_flush(new_buffer) + new_timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:noreply, %{buffer: [], timer: new_timer}} + else + {:noreply, %{state | buffer: new_buffer}} + end + end + + def handle_info(:tick, %{buffer: buffer}) do + do_flush(buffer) + timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:noreply, %{buffer: [], timer: timer}} + end + + def handle_call(:flush, _from, %{buffer: buffer} = state) do + Process.cancel_timer(state[:timer]) + do_flush(buffer) + new_timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:reply, nil, %{buffer: [], timer: new_timer}} + end + + def terminate(_reason, %{buffer: buffer}) do + Logger.info("Flushing event buffer before shutdown...") + do_flush(buffer) + end + + defp do_flush(buffer) do + case buffer do + [] -> + nil + + events -> + Logger.info("Flushing #{length(events)} events") + events = Enum.map(events, &(Map.from_struct(&1) |> Map.delete(:__meta__))) + + IngestRepo.insert_all(Plausible.ClickhouseEventV2, events) + end + end + + defp flush_interval_ms() do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms) + end + + defp max_buffer_size() do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size) end end diff --git a/lib/plausible/ingestion/write_buffer.ex b/lib/plausible/ingestion/write_buffer.ex deleted file mode 100644 index 7854357fb..000000000 --- a/lib/plausible/ingestion/write_buffer.ex +++ /dev/null @@ -1,150 +0,0 @@ -defmodule Plausible.Ingestion.WriteBuffer do - @moduledoc false - use GenServer - require Logger - - alias Plausible.IngestRepo - - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name)) - end - - def insert(server, row_binary) do - GenServer.cast(server, {:insert, row_binary}) - end - - def flush(server) do - GenServer.call(server, :flush, :infinity) - end - - @impl true - def init(opts) do - buffer = opts[:buffer] || [] - max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size() - flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms() - - Process.flag(:trap_exit, true) - timer = Process.send_after(self(), :tick, flush_interval_ms) - - {:ok, - %{ - buffer: buffer, - timer: timer, - name: Keyword.fetch!(opts, :name), - insert_sql: Keyword.fetch!(opts, :insert_sql), - insert_opts: Keyword.fetch!(opts, :insert_opts), - header: Keyword.fetch!(opts, :header), - buffer_size: IO.iodata_length(buffer), - max_buffer_size: max_buffer_size, - flush_interval_ms: flush_interval_ms - }} - end - - @impl true - - def handle_cast({:insert, row_binary}, state) do - state = %{ - state - | buffer: [state.buffer | row_binary], - buffer_size: state.buffer_size + IO.iodata_length(row_binary) - } - - if state.buffer_size >= state.max_buffer_size do - Logger.info("#{state.name} buffer full, flushing to ClickHouse") - Process.cancel_timer(state.timer) - do_flush(state) - new_timer = Process.send_after(self(), :tick, state.flush_interval_ms) - {:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}} - else - {:noreply, state} - end - end - - @impl true - def handle_info(:tick, state) do - do_flush(state) - timer = Process.send_after(self(), :tick, state.flush_interval_ms) - {:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}} - end - - @impl true - def handle_call(:flush, _from, state) do - %{timer: timer, flush_interval_ms: flush_interval_ms} = state - Process.cancel_timer(timer) - do_flush(state) - new_timer = Process.send_after(self(), :tick, flush_interval_ms) - {:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}} - end - - @impl true - def terminate(_reason, %{name: name} = state) do - Logger.info("Flushing #{name} buffer before shutdown...") - do_flush(state) - end - - defp do_flush(state) do - %{ - buffer: buffer, - buffer_size: buffer_size, - insert_opts: insert_opts, - insert_sql: insert_sql, - header: header, - name: name - } = state - - case buffer do - [] -> - nil - - _not_empty -> - Logger.info("Flushing #{buffer_size} byte(s) RowBinary from #{name}") - IngestRepo.query!(insert_sql, [header | buffer], insert_opts) - end - end - - defp default_flush_interval_ms do - Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms) - end - - defp default_max_buffer_size do - Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size) - end - - @doc false - def compile_time_prepare(schema) do - fields = schema.__schema__(:fields) - - types = - Enum.map(fields, fn field -> - type = schema.__schema__(:type, field) || raise "missing type for #{field}" - - type - |> Ecto.Type.type() - |> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field) - end) - - encoding_types = Ch.RowBinary.encoding_types(types) - - header = - fields - |> Enum.map(&to_string/1) - |> Ch.RowBinary.encode_names_and_types(types) - |> IO.iodata_to_binary() - - insert_sql = "INSERT INTO #{schema.__schema__(:source)} FORMAT RowBinaryWithNamesAndTypes" - - %{ - fields: fields, - types: types, - encoding_types: encoding_types, - header: header, - insert_sql: insert_sql, - insert_opts: [ - command: :insert, - encode: false, - source: schema.__schema__(:source), - cast_params: [] - ] - } - end -end diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.ex index 1adcbbb67..770fc965c 100644 --- a/lib/plausible/session/write_buffer.ex +++ b/lib/plausible/session/write_buffer.ex @@ -1,43 +1,83 @@ defmodule Plausible.Session.WriteBuffer do - @moduledoc false + use GenServer + require Logger - %{ - header: header, - insert_sql: insert_sql, - insert_opts: insert_opts, - fields: fields, - encoding_types: encoding_types - } = - Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseSessionV2) + alias Plausible.IngestRepo - def child_spec(opts) do - opts = - Keyword.merge(opts, - name: __MODULE__, - header: unquote(header), - insert_sql: unquote(insert_sql), - insert_opts: unquote(insert_opts) - ) + def start_link(_opts) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end - Plausible.Ingestion.WriteBuffer.child_spec(opts) + def init(buffer) do + Process.flag(:trap_exit, true) + timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:ok, %{buffer: buffer, timer: timer}} end def insert(sessions) do - row_binary = - sessions - |> Enum.map(fn %{is_bounce: is_bounce} = session -> - {:ok, is_bounce} = Plausible.ClickhouseSessionV2.BoolUInt8.dump(is_bounce) - session = %{session | is_bounce: is_bounce} - Enum.map(unquote(fields), fn field -> Map.fetch!(session, field) end) - end) - |> Ch.RowBinary._encode_rows(unquote(encoding_types)) - |> IO.iodata_to_binary() - - :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary) + GenServer.cast(__MODULE__, {:insert, sessions}) {:ok, sessions} end - def flush do - Plausible.Ingestion.WriteBuffer.flush(__MODULE__) + def flush() do + GenServer.call(__MODULE__, :flush, :infinity) + :ok + end + + def handle_cast({:insert, sessions}, %{buffer: buffer} = state) do + new_buffer = sessions ++ buffer + + if length(new_buffer) >= max_buffer_size() do + Logger.info("Buffer full, flushing to disk") + Process.cancel_timer(state[:timer]) + do_flush(new_buffer) + new_timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:noreply, %{buffer: [], timer: new_timer}} + else + {:noreply, %{state | buffer: new_buffer}} + end + end + + def handle_info(:tick, %{buffer: buffer}) do + do_flush(buffer) + timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:noreply, %{buffer: [], timer: timer}} + end + + def handle_call(:flush, _from, %{buffer: buffer} = state) do + Process.cancel_timer(state[:timer]) + do_flush(buffer) + new_timer = Process.send_after(self(), :tick, flush_interval_ms()) + {:reply, nil, %{buffer: [], timer: new_timer}} + end + + def terminate(_reason, %{buffer: buffer}) do + Logger.info("Flushing session buffer before shutdown...") + do_flush(buffer) + end + + defp do_flush(buffer) do + case buffer do + [] -> + nil + + sessions -> + Logger.info("Flushing #{length(sessions)} sessions") + + sessions = + sessions + |> Enum.map(&(Map.from_struct(&1) |> Map.delete(:__meta__))) + |> Enum.reverse() + + IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions) + end + end + + defp flush_interval_ms() do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms) + end + + defp max_buffer_size() do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size) end end