Switch to new clickhouse adapter (ch/chto) (#2733)
* another clickhouse adapter * don't restore stats_removal.ex * fix events main-graph error (#2746) * update ch, chto * update chto again (#2759) * Stop treating page filter as an entry_page filter (#2752) * remove dead code * stop treating page filter as entry page filter in breakdown queries * stop treating page filter as entry page filter in aggregate queries * stop treating page filter as entry page filter in timeseries queries * mix format * update changelog * break code down to smaller functions to keep credo happy * remove unused functions * make CSV export return only conversions with goal filter (#2760) * make CSV export return only conversions with goal filter * update changelog * update elixir version in mix.exs (#2742) * revert admin.ex changes (#2776) --------- Co-authored-by: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Co-authored-by: ruslandoga <rusl@n-do.ga> Co-authored-by: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com>pull/2778/head^2 r2023.03.21-79
parent
736e6e385c
commit
6d79ca5093
@ -1,10 +1,15 @@
|
||||
defmodule Mix.Tasks.CleanClickhouse do
|
||||
use Mix.Task
|
||||
|
||||
alias Plausible.IngestRepo
|
||||
|
||||
def run(_) do
|
||||
clean_events = "ALTER TABLE events DELETE WHERE 1"
|
||||
clean_sessions = "ALTER TABLE sessions DELETE WHERE 1"
|
||||
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_events)
|
||||
Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_sessions)
|
||||
%{rows: rows} = IngestRepo.query!("show tables")
|
||||
tables = Enum.map(rows, fn [table] -> table end)
|
||||
to_truncate = tables -- ["schema_migrations"]
|
||||
|
||||
Enum.each(to_truncate, fn table ->
|
||||
IngestRepo.query!("truncate #{table}")
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
@ -0,0 +1,148 @@
|
||||
defmodule Plausible.Google.ImportedVisitor do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_visitors" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :pageviews, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedSource do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_sources" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :source, :string
|
||||
field :utm_medium, :string
|
||||
field :utm_campaign, :string
|
||||
field :utm_content, :string
|
||||
field :utm_term, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedPage do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_pages" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :hostname, :string
|
||||
field :page, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :pageviews, Ch.Types.UInt64
|
||||
field :exits, Ch.Types.UInt64
|
||||
field :time_on_page, Ch.Types.UInt64
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedEntryPage do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_entry_pages" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :entry_page, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :entrances, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedExitPage do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_exit_pages" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :exit_page, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :exits, Ch.Types.UInt64
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedLocation do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_locations" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :country, :string
|
||||
field :region, :string
|
||||
field :city, Ch.Types.UInt64
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedDevice do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_devices" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :device, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedBrowser do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_browsers" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :browser, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Plausible.Google.ImportedOperatingSystem do
|
||||
@moduledoc false
|
||||
use Ecto.Schema
|
||||
|
||||
@primary_key false
|
||||
schema "imported_operating_systems" do
|
||||
field :site_id, Ch.Types.UInt64
|
||||
field :date, :date
|
||||
field :operating_system, :string
|
||||
field :visitors, Ch.Types.UInt64
|
||||
field :visits, Ch.Types.UInt64
|
||||
field :visit_duration, Ch.Types.UInt64
|
||||
field :bounces, Ch.Types.UInt32
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,4 @@
|
||||
[
|
||||
import_deps: [:ecto_sql],
|
||||
inputs: ["*.exs"]
|
||||
]
|
||||
@ -0,0 +1,242 @@
|
||||
CREATE TABLE plausible_events_db.sessions
|
||||
(
|
||||
`session_id` UInt64,
|
||||
`sign` Int8,
|
||||
`domain` String,
|
||||
`user_id` UInt64,
|
||||
`hostname` String,
|
||||
`is_bounce` UInt8,
|
||||
`entry_page` String,
|
||||
`exit_page` String,
|
||||
`pageviews` Int32,
|
||||
`events` Int32,
|
||||
`duration` UInt32,
|
||||
`referrer` String,
|
||||
`referrer_source` String,
|
||||
`country_code` LowCardinality(FixedString(2)),
|
||||
`screen_size` LowCardinality(String),
|
||||
`operating_system` LowCardinality(String),
|
||||
`browser` LowCardinality(String),
|
||||
`start` DateTime,
|
||||
`timestamp` DateTime,
|
||||
`utm_medium` String,
|
||||
`utm_source` String,
|
||||
`utm_campaign` String,
|
||||
`browser_version` LowCardinality(String),
|
||||
`operating_system_version` LowCardinality(String),
|
||||
`subdivision1_code` LowCardinality(String),
|
||||
`subdivision2_code` LowCardinality(String),
|
||||
`city_geoname_id` UInt32,
|
||||
`utm_content` String,
|
||||
`utm_term` String,
|
||||
`transferred_from` String,
|
||||
`entry_meta.key` Array(String),
|
||||
`entry_meta.value` Array(String)
|
||||
)
|
||||
ENGINE = CollapsingMergeTree(sign)
|
||||
PARTITION BY toYYYYMM(start)
|
||||
ORDER BY (domain, toDate(start), user_id, session_id)
|
||||
SAMPLE BY user_id
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.ingest_counters
|
||||
(
|
||||
`event_timebucket` DateTime,
|
||||
`domain` LowCardinality(String),
|
||||
`site_id` Nullable(UInt64),
|
||||
`metric` LowCardinality(String),
|
||||
`value` UInt64
|
||||
)
|
||||
ENGINE = SummingMergeTree(value)
|
||||
ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_visitors
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`visitors` UInt64,
|
||||
`pageviews` UInt64,
|
||||
`bounces` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_sources
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`source` String,
|
||||
`utm_medium` String,
|
||||
`utm_campaign` String,
|
||||
`utm_content` String,
|
||||
`utm_term` String,
|
||||
`visitors` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, source)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_pages
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`hostname` String,
|
||||
`page` String,
|
||||
`visitors` UInt64,
|
||||
`pageviews` UInt64,
|
||||
`exits` UInt64,
|
||||
`time_on_page` UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, hostname, page)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_operating_systems
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`operating_system` String,
|
||||
`visitors` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, operating_system)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_locations
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`country` String,
|
||||
`region` String,
|
||||
`city` UInt64,
|
||||
`visitors` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, country, region, city)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_exit_pages
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`exit_page` String,
|
||||
`visitors` UInt64,
|
||||
`exits` UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, exit_page)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_entry_pages
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`entry_page` String,
|
||||
`visitors` UInt64,
|
||||
`entrances` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, entry_page)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_devices
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`device` String,
|
||||
`visitors` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, device)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.imported_browsers
|
||||
(
|
||||
`site_id` UInt64,
|
||||
`date` Date,
|
||||
`browser` String,
|
||||
`visitors` UInt64,
|
||||
`visits` UInt64,
|
||||
`visit_duration` UInt64,
|
||||
`bounces` UInt32
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (site_id, date, browser)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.events
|
||||
(
|
||||
`name` String,
|
||||
`domain` String,
|
||||
`user_id` UInt64,
|
||||
`session_id` UInt64,
|
||||
`hostname` String,
|
||||
`pathname` String,
|
||||
`referrer` String,
|
||||
`referrer_source` String,
|
||||
`country_code` LowCardinality(FixedString(2)),
|
||||
`screen_size` LowCardinality(String),
|
||||
`operating_system` LowCardinality(String),
|
||||
`browser` LowCardinality(String),
|
||||
`timestamp` DateTime,
|
||||
`utm_medium` String,
|
||||
`utm_source` String,
|
||||
`utm_campaign` String,
|
||||
`meta.key` Array(String),
|
||||
`meta.value` Array(String),
|
||||
`browser_version` LowCardinality(String),
|
||||
`operating_system_version` LowCardinality(String),
|
||||
`subdivision1_code` LowCardinality(String),
|
||||
`subdivision2_code` LowCardinality(String),
|
||||
`city_geoname_id` UInt32,
|
||||
`utm_content` String,
|
||||
`utm_term` String,
|
||||
`transferred_from` String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (domain, toDate(timestamp), user_id)
|
||||
SAMPLE BY user_id
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
CREATE TABLE plausible_events_db.schema_migrations
|
||||
(
|
||||
`version` Int64,
|
||||
`inserted_at` DateTime
|
||||
)
|
||||
ENGINE = TinyLog;
|
||||
|
||||
INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES
|
||||
(20200915070607,'2023-03-08 10:03:33'),
|
||||
(20200918075025,'2023-03-08 10:03:33'),
|
||||
(20201020083739,'2023-03-08 10:03:33'),
|
||||
(20201106125234,'2023-03-08 10:03:33'),
|
||||
(20210323130440,'2023-03-08 10:03:33'),
|
||||
(20210712214034,'2023-03-08 10:03:33'),
|
||||
(20211017093035,'2023-03-08 10:03:33'),
|
||||
(20211112130238,'2023-03-08 10:03:33'),
|
||||
(20220310104931,'2023-03-08 10:03:33'),
|
||||
(20220404123000,'2023-03-08 10:03:33'),
|
||||
(20220421161259,'2023-03-08 10:03:33'),
|
||||
(20220422075510,'2023-03-08 10:03:33'),
|
||||
(20230124140348,'2023-03-08 10:03:33'),
|
||||
(20230210140348,'2023-03-08 10:03:33'),
|
||||
(20230214114402,'2023-03-08 10:03:33');
|
||||
Loading…
Reference in New Issue