Split clickhouse repos - making the main one read only (#2826)
* Split clickhouse pools into readonly/import deletions * Remove CRM site transfers * Initialize ImportDeletionRepo * Put ImportDeletionRepo to usepull/2832/head r2023.04.06-97
parent
509c549c01
commit
154ce3a44c
@ -0,0 +1,17 @@
|
||||
defmodule Plausible.ImportDeletionRepo do
|
||||
@moduledoc """
|
||||
A dedicated repo for import related mutations
|
||||
"""
|
||||
|
||||
use Ecto.Repo,
|
||||
otp_app: :plausible,
|
||||
adapter: Ecto.Adapters.ClickHouse
|
||||
|
||||
defmacro __using__(_) do
|
||||
quote do
|
||||
alias Plausible.ImportDeletionRepo
|
||||
import Ecto
|
||||
import Ecto.Query, only: [from: 1, from: 2]
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,130 +0,0 @@
|
||||
defmodule Plausible.SiteAdminTest do
|
||||
use Plausible.DataCase, async: true
|
||||
|
||||
alias Plausible.{
|
||||
SiteAdmin,
|
||||
ClickhouseRepo,
|
||||
ClickhouseEvent,
|
||||
ClickhouseSession
|
||||
}
|
||||
|
||||
describe "transfer_data" do
|
||||
test "event and session structs remain the same after transfer" do
|
||||
if Plausible.v2?() do
|
||||
:ok
|
||||
else
|
||||
from_site = insert(:site)
|
||||
to_site = insert(:site)
|
||||
|
||||
populate_stats(from_site, [build(:pageview)])
|
||||
|
||||
event_before = get_event_by_domain(from_site.domain)
|
||||
session_before = get_session_by_domain(from_site.domain)
|
||||
|
||||
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
|
||||
|
||||
event_after = get_event_by_domain(to_site.domain)
|
||||
session_after = get_session_by_domain(to_site.domain)
|
||||
|
||||
assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
|
||||
assert event_before == %ClickhouseEvent{event_after | transferred_from: ""}
|
||||
assert session_before == %ClickhouseSession{session_after | transferred_from: ""}
|
||||
assert event_after.transferred_from == from_site.domain
|
||||
assert session_after.transferred_from == from_site.domain
|
||||
end
|
||||
end
|
||||
|
||||
test "transfers all events and sessions" do
|
||||
if Plausible.v2?() do
|
||||
:ok
|
||||
else
|
||||
from_site = insert(:site)
|
||||
to_site = insert(:site)
|
||||
|
||||
populate_stats(from_site, [
|
||||
build(:pageview, user_id: 123),
|
||||
build(:event, name: "Signup", user_id: 123),
|
||||
build(:pageview, user_id: 456),
|
||||
build(:event, name: "Signup", user_id: 789)
|
||||
])
|
||||
|
||||
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
|
||||
|
||||
transferred_events =
|
||||
ClickhouseRepo.all(
|
||||
from e in Plausible.ClickhouseEvent, where: e.domain == ^to_site.domain
|
||||
)
|
||||
|
||||
transferred_sessions =
|
||||
ClickhouseRepo.all(
|
||||
from e in Plausible.ClickhouseSession, where: e.domain == ^to_site.domain
|
||||
)
|
||||
|
||||
assert length(transferred_events) == 4
|
||||
assert length(transferred_sessions) == 3
|
||||
end
|
||||
end
|
||||
|
||||
test "updates stats_start_date on site record" do
|
||||
if Plausible.v2?() do
|
||||
:ok
|
||||
else
|
||||
from_site = insert(:site)
|
||||
to_site = insert(:site)
|
||||
|
||||
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
|
||||
|
||||
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
|
||||
|
||||
assert Repo.reload(to_site).stats_start_date == ~D[2022-01-01]
|
||||
end
|
||||
end
|
||||
|
||||
test "updates native_stats_start_date on based on the from_site record" do
|
||||
if Plausible.v2?() do
|
||||
:ok
|
||||
else
|
||||
from_site = insert(:site, native_stats_start_at: ~N[2022-01-01 01:00:00])
|
||||
to_site = insert(:site)
|
||||
|
||||
populate_stats(from_site, [build(:pageview, timestamp: ~N[2022-01-01 13:21:00])])
|
||||
|
||||
SiteAdmin.transfer_data([from_site], %{"domain" => to_site.domain})
|
||||
|
||||
assert Repo.reload(to_site).native_stats_start_at == ~N[2022-01-01 01:00:00]
|
||||
end
|
||||
end
|
||||
|
||||
test "session_transfer_query" do
|
||||
actual = SiteAdmin.session_transfer_query("from.com", "to.com")
|
||||
|
||||
expected =
|
||||
"INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')"
|
||||
|
||||
assert actual == expected
|
||||
end
|
||||
|
||||
test "event_transfer_query" do
|
||||
actual = SiteAdmin.event_transfer_query("from.com", "to.com")
|
||||
|
||||
expected =
|
||||
"INSERT INTO events (browser, browser_version, city_geoname_id, country_code, domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, hostname, meta.key, meta.value, name, operating_system, operating_system_version, pathname, referrer, referrer_source, screen_size, session_id, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM events WHERE domain='from.com')"
|
||||
|
||||
assert actual == expected
|
||||
end
|
||||
end
|
||||
|
||||
defp get_event_by_domain(domain) do
|
||||
q = from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
|
||||
|
||||
Plausible.ClickhouseRepo.one!(q)
|
||||
|> Map.drop([:__meta__, :domain])
|
||||
end
|
||||
|
||||
defp get_session_by_domain(domain) do
|
||||
q = from s in Plausible.ClickhouseSession, where: s.domain == ^domain
|
||||
|
||||
Plausible.ClickhouseRepo.one!(q)
|
||||
|> Map.drop([:__meta__, :domain])
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue