From 154ce3a44cbd06c012950abd924238d095f42cda Mon Sep 17 00:00:00 2001 From: hq1 Date: Thu, 6 Apr 2023 12:45:36 +0200 Subject: [PATCH] Split clickhouse repos - making the main one read only (#2826) * Split clickhouse pools into readonly/import deletions * Remove CRM site transfers * Initialize ImportDeletionRepo * Put ImportDeletionRepo to use --- config/runtime.exs | 13 ++- lib/plausible/application.ex | 1 + lib/plausible/import_deletion_repo.ex | 17 ++++ lib/plausible/purge.ex | 2 +- lib/plausible/site/admin.ex | 77 --------------- test/plausible/site/admin_test.exs | 130 -------------------------- 6 files changed, 31 insertions(+), 209 deletions(-) create mode 100644 lib/plausible/import_deletion_repo.ex delete mode 100644 test/plausible/site/admin_test.exs diff --git a/config/runtime.exs b/config/runtime.exs index 3151029a4..50804dd25 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -289,7 +289,10 @@ config :plausible, Plausible.ClickhouseRepo, queue_target: 500, queue_interval: 2000, url: ch_db_url, - transport_opts: ch_transport_opts + transport_opts: ch_transport_opts, + settings: [ + readonly: 1 + ] config :plausible, Plausible.IngestRepo, loggers: [Ecto.LogEntry], @@ -313,6 +316,14 @@ config :plausible, Plausible.AsyncInsertRepo, wait_for_async_insert: 0 ] +config :plausible, Plausible.ImportDeletionRepo, + loggers: [Ecto.LogEntry], + queue_target: 500, + queue_interval: 2000, + url: ch_db_url, + transport_opts: ch_transport_opts, + pool_size: 1 + case mailer_adapter do "Bamboo.PostmarkAdapter" -> config :plausible, Plausible.Mailer, diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 27c1d697f..b2f75f8f6 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -11,6 +11,7 @@ defmodule Plausible.Application do Plausible.ClickhouseRepo, Plausible.IngestRepo, Plausible.AsyncInsertRepo, + Plausible.ImportDeletionRepo, Plausible.Ingestion.Counters, {Finch, name: Plausible.Finch, pools: finch_pool_config()}, {Phoenix.PubSub, name: Plausible.PubSub}, diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/import_deletion_repo.ex new file mode 100644 index 000000000..3c8a012d8 --- /dev/null +++ b/lib/plausible/import_deletion_repo.ex @@ -0,0 +1,17 @@ +defmodule Plausible.ImportDeletionRepo do + @moduledoc """ + A dedicated repo for import related mutations + """ + + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.ClickHouse + + defmacro __using__(_) do + quote do + alias Plausible.ImportDeletionRepo + import Ecto + import Ecto.Query, only: [from: 1, from: 2] + end + end +end diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex index a075108c4..4288f74b5 100644 --- a/lib/plausible/purge.ex +++ b/lib/plausible/purge.ex @@ -17,7 +17,7 @@ defmodule Plausible.Purge do def delete_imported_stats!(site) do Enum.each(Plausible.Imported.tables(), fn table -> sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}" - Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id]) + Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id]) end) clear_stats_start_date!(site) diff --git a/lib/plausible/site/admin.ex b/lib/plausible/site/admin.ex index 032f19662..6333b1d7d 100644 --- a/lib/plausible/site/admin.ex +++ b/lib/plausible/site/admin.ex @@ -53,18 +53,6 @@ defmodule Plausible.SiteAdmin do ] end - def list_actions(_conn) do - [ - transfer_data: %{ - name: "Transfer data", - inputs: [ - %{name: "domain", title: "to domain", default: nil} - ], - action: fn _conn, sites, params -> transfer_data(sites, params) end - } - ] - end - defp format_date(date) do Timex.format!(date, "{Mshort} {D}, {YYYY}") end @@ -83,58 +71,6 @@ defmodule Plausible.SiteAdmin do |> Enum.join(", ") end - def transfer_data([from_site], params) do - if Plausible.v2?() do - {:error, "Transfers temporarily unspported with v2"} - else - to_site = Repo.get_by(Plausible.Site, domain: params["domain"]) - - if to_site do - event_q = event_transfer_query(from_site.domain, to_site.domain) - {:ok, _} = Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, event_q, [], timeout: 30_000) - - session_q = session_transfer_query(from_site.domain, to_site.domain) - - {:ok, _} = - Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, session_q, [], timeout: 30_000) - - start_date = Plausible.Stats.Clickhouse.pageview_start_date_local(from_site) - - {:ok, _} = - to_site - |> Plausible.Site.set_stats_start_date(start_date) - |> Plausible.Site.set_native_stats_start_at(from_site.native_stats_start_at) - |> Repo.update() - - :ok - else - {:error, "Cannot transfer to non-existing domain"} - end - end - end - - def transfer_data(_, _), do: {:error, "Please select exactly one site for this action"} - - def session_transfer_query(from_domain, to_domain) do - fields = get_struct_fields(Plausible.ClickhouseSession) - - "INSERT INTO sessions (" <> - stringify_fields(fields) <> - ") SELECT " <> - stringify_fields(fields, to_domain, from_domain) <> - " FROM (SELECT * FROM sessions WHERE domain='#{from_domain}')" - end - - def event_transfer_query(from_domain, to_domain) do - fields = get_struct_fields(Plausible.ClickhouseEvent) - - "INSERT INTO events (" <> - stringify_fields(fields) <> - ") SELECT " <> - stringify_fields(fields, to_domain, from_domain) <> - " FROM (SELECT * FROM events WHERE domain='#{from_domain}')" - end - def get_struct_fields(module) do module.__struct__() |> Map.drop([:__meta__, :__struct__]) @@ -143,19 +79,6 @@ defmodule Plausible.SiteAdmin do |> Enum.sort() end - defp stringify_fields(fields), do: Enum.join(fields, ", ") - - defp stringify_fields(fields, domain_value, transferred_from_value) do - Enum.map(fields, fn field -> - case field do - "domain" -> "'#{domain_value}' as domain" - "transferred_from" -> "'#{transferred_from_value}' as transferred_from" - _ -> field - end - end) - |> stringify_fields() - end - def create_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs) def update_changeset(schema, attrs), do: Plausible.Site.crm_changeset(schema, attrs) end diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs deleted file mode 100644 index 99d5429c6..000000000 --- a/test/plausible/site/admin_test.exs +++ /dev/null @@ -1,130 +0,0 @@ -defmodule Plausible.SiteAdminTest do - use Plausible.DataCase, async: true - - alias Plausible.{ - SiteAdmin, - ClickhouseRepo, - ClickhouseEvent, - ClickhouseSession - } - - describe "transfer_data" do - test "event and session structs remain the same after transfer" do - if Plausible.v2?() do - :ok - else - from_site = insert(:site) - to_site = insert(:site) - - populate_stats(from_site, [build(:pageview)]) - - event_before = get_event_by_domain(from_site.domain) - session_before = get_session_by_domain(from_site.domain) - - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - - event_after = get_event_by_domain(to_site.domain) - session_after = get_session_by_domain(to_site.domain) - - assert event_before == %ClickhouseEvent{event_after | transferred_from: ""} - assert event_before == %ClickhouseEvent{event_after | transferred_from: ""} - assert session_before == %ClickhouseSession{session_after | transferred_from: ""} - assert event_after.transferred_from == from_site.domain - assert session_after.transferred_from == from_site.domain - end - end - - test "transfers all events and sessions" do - if Plausible.v2?() do - :ok - else - from_site = insert(:site) - to_site = insert(:site) - - populate_stats(from_site, [ - build(:pageview, user_id: 123), - build(:event, name: "Signup", user_id: 123), - build(:pageview, user_id: 456), - build(:event, name: "Signup", user_id: 789) - ]) - - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - - transferred_events = - ClickhouseRepo.all( - from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain - ) - - transferred_sessions = - ClickhouseRepo.all( - from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain - ) - - assert length(transferred_events) == 4 - assert length(transferred_sessions) == 3 - end - end - - test "updates stats_start_date on site record" do - if Plausible.v2?() do - :ok - else - from_site = insert(:site) - to_site = insert(:site) - - populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])]) - - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - - assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01] - end - end - - test "updates native_stats_start_date on based on the from_site record" do - if Plausible.v2?() do - :ok - else - from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00]) - to_site = insert(:site) - - populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])]) - - SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain}) - - assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00] - end - end - - test "session_transfer_query" do - actual = SiteAdmin.session_transfer_query("from.com", "to.com") - - expected = - "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" - - assert actual == expected - end - - test "event_transfer_query" do - actual = SiteAdmin.event_transfer_query("from.com", "to.com") - - expected = - "INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')" - - assert actual == expected - end - end - - defp get_event_by_domain(domain) do - q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain - - Plausible.ClickhouseRepo.one!(q) - |> Map.drop([:__meta__, :domain]) - end - - defp get_session_by_domain(domain) do - q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain - - Plausible.ClickhouseRepo.one!(q) - |> Map.drop([:__meta__, :domain]) - end -end