diff --git a/config/runtime.exs b/config/runtime.exs index 5cc3882ba..744b66e1c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -296,7 +296,7 @@ config :plausible, Plausible.AsyncInsertRepo, queue_interval: 2000, url: ch_db_url, pool_size: 1, - clickhouse_settings: [ + settings: [ async_insert: 1, wait_for_async_insert: 0 ] diff --git a/lib/mix/tasks/clean_clickhouse.ex b/lib/mix/tasks/clean_clickhouse.ex index 02c4c97c8..71b984b49 100644 --- a/lib/mix/tasks/clean_clickhouse.ex +++ b/lib/mix/tasks/clean_clickhouse.ex @@ -1,10 +1,15 @@ defmodule Mix.Tasks.CleanClickhouse do use Mix.Task + alias Plausible.IngestRepo + def run(_) do - clean_events = "ALTER TABLE events DELETE WHERE 1" - clean_sessions = "ALTER TABLE sessions DELETE WHERE 1" - Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_events) - Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, clean_sessions) + %{rows: rows} = IngestRepo.query!("show tables") + tables = Enum.map(rows, fn [table] -> table end) + to_truncate = tables -- ["schema_migrations"] + + Enum.each(to_truncate, fn table -> + IngestRepo.query!("truncate #{table}") + end) end end diff --git a/lib/plausible/async_insert_repo.ex b/lib/plausible/async_insert_repo.ex index 6fc2e931c..d73bbe8ff 100644 --- a/lib/plausible/async_insert_repo.ex +++ b/lib/plausible/async_insert_repo.ex @@ -5,7 +5,7 @@ defmodule Plausible.AsyncInsertRepo do use Ecto.Repo, otp_app: :plausible, - adapter: ClickhouseEcto + adapter: Ecto.Adapters.ClickHouse defmacro __using__(_) do quote do diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index cc8f5f206..f6f8631d9 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -1,7 +1,7 @@ defmodule Plausible.ClickhouseRepo do use Ecto.Repo, otp_app: :plausible, - adapter: ClickhouseEcto, + adapter: Ecto.Adapters.ClickHouse, read_only: true defmacro __using__(_) do diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.ex index cb957576a..ae8ab148a 100644 --- a/lib/plausible/event/clickhouse_schema.ex +++ b/lib/plausible/event/clickhouse_schema.ex @@ -8,32 +8,32 @@ defmodule Plausible.ClickhouseEvent do field :domain, :string field :hostname, :string field :pathname, :string - field :user_id, :integer - field :session_id, :integer + field :user_id, Ch.Types.UInt64 + field :session_id, Ch.Types.UInt64 field :timestamp, :naive_datetime - field :referrer, :string, default: "" - field :referrer_source, :string, default: "" - field :utm_medium, :string, default: "" - field :utm_source, :string, default: "" - field :utm_campaign, :string, default: "" - field :utm_content, :string, default: "" - field :utm_term, :string, default: "" + field :referrer, :string + field :referrer_source, :string + field :utm_medium, :string + field :utm_source, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string - field :country_code, :string, default: "" - field :subdivision1_code, :string, default: "" - field :subdivision2_code, :string, default: "" - field :city_geoname_id, :integer, default: 0 + field :country_code, Ch.Types.FixedString, size: 2 + field :subdivision1_code, :string + field :subdivision2_code, :string + field :city_geoname_id, Ch.Types.UInt32 - field :screen_size, :string, default: "" - field :operating_system, :string, default: "" - field :operating_system_version, :string, default: "" - field :browser, :string, default: "" - field :browser_version, :string, default: "" + field :screen_size, :string + field :operating_system, :string + field :operating_system_version, :string + field :browser, :string + field :browser_version, :string - field :"meta.key", {:array, :string}, default: [] - field :"meta.value", {:array, :string}, default: [] - field :transferred_from, :string, default: "" + field :"meta.key", {:array, :string} + field :"meta.value", {:array, :string} + field :transferred_from, :string end def new(attrs) do diff --git a/lib/plausible/google/buffer.ex b/lib/plausible/google/buffer.ex index b43160dfa..a32679fec 100644 --- a/lib/plausible/google/buffer.ex +++ b/lib/plausible/google/buffer.ex @@ -91,6 +91,23 @@ defmodule Plausible.Google.Buffer do Process.sleep(1000) Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer") - Plausible.IngestRepo.insert_all(table_name, records) + insert_all(table_name, records) end + + # used in tests `setup` + @doc false + def insert_all(table_name, records) do + schema = table_schema(table_name) + Plausible.IngestRepo.insert_all(schema, records) + end + + defp table_schema("imported_visitors"), do: Plausible.Google.ImportedVisitor + defp table_schema("imported_sources"), do: Plausible.Google.ImportedSource + defp table_schema("imported_pages"), do: Plausible.Google.ImportedPage + defp table_schema("imported_entry_pages"), do: Plausible.Google.ImportedEntryPage + defp table_schema("imported_exit_pages"), do: Plausible.Google.ImportedExitPage + defp table_schema("imported_locations"), do: Plausible.Google.ImportedLocation + defp table_schema("imported_devices"), do: Plausible.Google.ImportedDevice + defp table_schema("imported_browsers"), do: Plausible.Google.ImportedBrowser + defp table_schema("imported_operating_systems"), do: Plausible.Google.ImportedOperatingSystem end diff --git a/lib/plausible/google/schemas.ex b/lib/plausible/google/schemas.ex new file mode 100644 index 000000000..780b692cc --- /dev/null +++ b/lib/plausible/google/schemas.ex @@ -0,0 +1,148 @@ +defmodule Plausible.Google.ImportedVisitor do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_visitors" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :visitors, Ch.Types.UInt64 + field :pageviews, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + end +end + +defmodule Plausible.Google.ImportedSource do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_sources" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :source, :string + field :utm_medium, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string + field :visitors, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end + +defmodule Plausible.Google.ImportedPage do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_pages" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :hostname, :string + field :page, :string + field :visitors, Ch.Types.UInt64 + field :pageviews, Ch.Types.UInt64 + field :exits, Ch.Types.UInt64 + field :time_on_page, Ch.Types.UInt64 + end +end + +defmodule Plausible.Google.ImportedEntryPage do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_entry_pages" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :entry_page, :string + field :visitors, Ch.Types.UInt64 + field :entrances, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end + +defmodule Plausible.Google.ImportedExitPage do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_exit_pages" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :exit_page, :string + field :visitors, Ch.Types.UInt64 + field :exits, Ch.Types.UInt64 + end +end + +defmodule Plausible.Google.ImportedLocation do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_locations" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :country, :string + field :region, :string + field :city, Ch.Types.UInt64 + field :visitors, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end + +defmodule Plausible.Google.ImportedDevice do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_devices" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :device, :string + field :visitors, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end + +defmodule Plausible.Google.ImportedBrowser do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_browsers" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :browser, :string + field :visitors, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end + +defmodule Plausible.Google.ImportedOperatingSystem do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_operating_systems" do + field :site_id, Ch.Types.UInt64 + field :date, :date + field :operating_system, :string + field :visitors, Ch.Types.UInt64 + field :visits, Ch.Types.UInt64 + field :visit_duration, Ch.Types.UInt64 + field :bounces, Ch.Types.UInt32 + end +end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 6b319df01..6e30daaef 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -31,7 +31,7 @@ defmodule Plausible.Imported do defp parse_number(nr) do {float, ""} = Float.parse(nr) - float + round(float) end defp new_from_google_analytics(site_id, "imported_visitors", row) do diff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.ex index 7928bd901..3fe099f10 100644 --- a/lib/plausible/ingest_repo.ex +++ b/lib/plausible/ingest_repo.ex @@ -5,7 +5,7 @@ defmodule Plausible.IngestRepo do use Ecto.Repo, otp_app: :plausible, - adapter: ClickhouseEcto + adapter: Ecto.Adapters.ClickHouse defmacro __using__(_) do quote do diff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.ex index 8c4b70aae..ad1532766 100644 --- a/lib/plausible/ingestion/counters.ex +++ b/lib/plausible/ingestion/counters.ex @@ -14,7 +14,7 @@ defmodule Plausible.Ingestion.Counters do aggregate into a 1-minute resolution. Clickhouse connection is set to insert counters asynchronously every time - a pool checkout is made. Those properties are reverted once the insert is done + a pool checkout is made. Those properties are reverted once the insert is done (or naturally, if the connection crashes). """ diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.ex index 1e4c8fe1f..941be5c9a 100644 --- a/lib/plausible/ingestion/counters/record.ex +++ b/lib/plausible/ingestion/counters/record.ex @@ -9,9 +9,9 @@ defmodule Plausible.Ingestion.Counters.Record do @primary_key false schema "ingest_counters" do field :event_timebucket, :utc_datetime - field :site_id, :integer + field :site_id, Ch.Types.Nullable, type: Ch.Types.UInt64 field :domain, :string field :metric, :string - field :value, :integer + field :value, Ch.Types.UInt64 end end diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex index a4be39213..a075108c4 100644 --- a/lib/plausible/purge.ex +++ b/lib/plausible/purge.ex @@ -16,7 +16,7 @@ defmodule Plausible.Purge do """ def delete_imported_stats!(site) do Enum.each(Plausible.Imported.tables(), fn table -> - sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?" + sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}" Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [site.id]) end) diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index fe342a573..b48e1bc22 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -53,41 +53,16 @@ defmodule Plausible.Session.CacheStore do duration: Timex.diff(event.timestamp, session.start, :second) |> abs, pageviews: if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews), - country_code: - if(session.country_code == "", do: event.country_code, else: session.country_code), - subdivision1_code: - if(session.subdivision1_code == "", - do: event.subdivision1_code, - else: session.subdivision1_code - ), - subdivision2_code: - if(session.subdivision2_code == "", - do: event.subdivision2_code, - else: session.subdivision2_code - ), - city_geoname_id: - if(session.city_geoname_id == 0, - do: event.city_geoname_id, - else: session.city_geoname_id - ), - operating_system: - if(session.operating_system == "", - do: event.operating_system, - else: session.operating_system - ), + country_code: session.country_code || event.country_code, + subdivision1_code: session.subdivision1_code || event.subdivision1_code, + subdivision2_code: session.subdivision2_code || event.subdivision2_code, + city_geoname_id: session.city_geoname_id || event.city_geoname_id, + operating_system: session.operating_system || event.operating_system, operating_system_version: - if(session.operating_system_version == "", - do: event.operating_system_version, - else: session.operating_system_version - ), - browser: if(session.browser == "", do: event.browser, else: session.browser), - browser_version: - if(session.browser_version == "", - do: event.browser_version, - else: session.browser_version - ), - screen_size: - if(session.screen_size == "", do: event.screen_size, else: session.screen_size), + session.operating_system_version || event.operating_system_version, + browser: session.browser || event.browser, + browser_version: session.browser_version || event.browser_version, + screen_size: session.screen_size || event.screen_size, events: session.events + 1 } end diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index 3645fd0f9..256fac3fe 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -6,20 +6,20 @@ defmodule Plausible.ClickhouseSession do schema "sessions" do field :hostname, :string field :domain, :string - field :user_id, :integer - field :session_id, :integer + field :user_id, Ch.Types.UInt64 + field :session_id, Ch.Types.UInt64 field :start, :naive_datetime - field :duration, :integer + field :duration, Ch.Types.UInt32 field :is_bounce, :boolean field :entry_page, :string field :exit_page, :string - field :pageviews, :integer - field :events, :integer - field :sign, :integer + field :pageviews, Ch.Types.Int32 + field :events, Ch.Types.Int32 + field :sign, Ch.Types.Int8 - field :"entry_meta.key", {:array, :string}, default: [] - field :"entry_meta.value", {:array, :string}, default: [] + field :"entry_meta.key", {:array, :string} + field :"entry_meta.value", {:array, :string} field :utm_medium, :string field :utm_source, :string @@ -29,10 +29,10 @@ defmodule Plausible.ClickhouseSession do field :referrer, :string field :referrer_source, :string - field :country_code, :string, default: "" - field :subdivision1_code, :string, default: "" - field :subdivision2_code, :string, default: "" - field :city_geoname_id, :integer, default: 0 + field :country_code, Ch.Types.FixedString, size: 2 + field :subdivision1_code, :string + field :subdivision2_code, :string + field :city_geoname_id, Ch.Types.UInt32 field :screen_size, :string field :operating_system, :string @@ -41,7 +41,7 @@ defmodule Plausible.ClickhouseSession do field :browser_version, :string field :timestamp, :naive_datetime - field :transferred_from, :string, default: "" + field :transferred_from, :string end def random_uint64() do diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 146d9c644..4b913c4d3 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -61,24 +61,28 @@ defmodule Plausible.Stats.Aggregate do ) {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) + where_param_idx = length(base_query_raw_params) + where_param = "{$#{where_param_idx}:String}" {where_clause, where_arg} = case query.filters["event:page"] do {:is, page} -> - {"p = ?", page} + {"p = #{where_param}", page} {:is_not, page} -> - {"p != ?", page} + {"p != #{where_param}", page} {:matches, expr} -> regex = page_regex(expr) - {"match(p, ?)", regex} + {"match(p, #{where_param})", regex} {:does_not_match, expr} -> regex = page_regex(expr) - {"not(match(p, ?))", regex} + {"not(match(p, #{where_param}))", regex} end + params = base_query_raw_params ++ [where_arg] + time_query = " SELECT avg(ifNotFinite(avgTime, null)) @@ -102,7 +106,7 @@ defmodule Plausible.Stats.Aggregate do GROUP BY p,p2,s) GROUP BY p)" - {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg]) + {:ok, res} = ClickhouseRepo.query(time_query, params) [[time_on_page]] = res.rows %{time_on_page: time_on_page} end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 0a464c65e..244f57640 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -174,7 +174,7 @@ defmodule Plausible.Stats.Base do {:member, values} -> list = Enum.map(values, &db_prop_val(prop_name, &1)) - from(s in sessions_q, where: fragment("? in tuple(?)", field(s, ^prop_name), ^list)) + from(s in sessions_q, where: field(s, ^prop_name) in ^list) {:matches, expr} -> regex = page_regex(expr) @@ -399,8 +399,10 @@ defmodule Plausible.Stats.Base do NaiveDateTime.utc_now() |> Timex.shift(seconds: 5) |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) - first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) + first_datetime = + NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) |> NaiveDateTime.truncate(:second) {first_datetime, last_datetime} end @@ -410,8 +412,10 @@ defmodule Plausible.Stats.Base do NaiveDateTime.utc_now() |> Timex.shift(seconds: 5) |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) - first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) + first_datetime = + NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) |> NaiveDateTime.truncate(:second) {first_datetime, last_datetime} end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 0abe33552..66f0ea07b 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -44,14 +44,14 @@ defmodule Plausible.Stats.Breakdown do offset: ^offset, where: fragment( - "notEmpty(multiMatchAllIndices(?, array(?)) as indices)", + "notEmpty(multiMatchAllIndices(?, ?) as indices)", e.pathname, ^page_regexes ), group_by: fragment("index"), select: %{ index: fragment("arrayJoin(indices) as index"), - goal: fragment("concat('Visit ', array(?)[index])", ^page_exprs) + goal: fragment("concat('Visit ', ?[index])", ^page_exprs) } ) |> select_event_metrics(metrics) @@ -242,6 +242,9 @@ defmodule Plausible.Stats.Breakdown do "round(sum(td)/count(case when p2 != p then 1 end))" end + pages_idx = length(base_query_raw_params) + params = base_query_raw_params ++ [pages] + time_query = " SELECT p, @@ -258,11 +261,11 @@ defmodule Plausible.Stats.Breakdown do neighbor(p, 1) as p2, neighbor(s, 1) as s2 FROM (#{base_query_raw})) - WHERE s=s2 AND p IN tuple(?) + WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)} GROUP BY p,p2,s) GROUP BY p" - {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages]) + {:ok, res} = ClickhouseRepo.query(time_query, params) if query.include_imported do # Imported page views have pre-calculated values @@ -367,7 +370,7 @@ defmodule Plausible.Stats.Breakdown do group_by: fragment("index"), select_merge: %{ index: fragment("arrayJoin(indices) as index"), - page_match: fragment("array(?)[index]", ^match_exprs) + page_match: fragment("?[index]", ^match_exprs) }, order_by: {:asc, fragment("index")} ) diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 5ca3551c9..a23b48d65 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -176,7 +176,7 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.all( from e in "events", group_by: e.domain, - where: fragment("? IN tuple(?)", e.domain, ^domains), + where: e.domain in ^domains, where: e.timestamp > fragment("now() - INTERVAL 24 HOUR"), select: {e.domain, fragment("uniq(user_id)")} ) @@ -421,7 +421,7 @@ defmodule Plausible.Stats.Clickhouse do else from( e in q, - inner_lateral_join: meta in fragment("meta as m"), + inner_lateral_join: meta in fragment("meta"), where: meta.key == ^key and meta.value == ^val ) end @@ -435,19 +435,25 @@ defmodule Plausible.Stats.Clickhouse do end defp utc_boundaries(%Query{period: "30m"}, site) do - last_datetime = NaiveDateTime.utc_now() + last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) first_datetime = - last_datetime |> Timex.shift(minutes: -30) |> beginning_of_time(site.native_stats_start_at) + last_datetime + |> Timex.shift(minutes: -30) + |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) {first_datetime, last_datetime} end defp utc_boundaries(%Query{period: "realtime"}, site) do - last_datetime = NaiveDateTime.utc_now() + last_datetime = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) first_datetime = - last_datetime |> Timex.shift(minutes: -5) |> beginning_of_time(site.native_stats_start_at) + last_datetime + |> Timex.shift(minutes: -5) + |> beginning_of_time(site.native_stats_start_at) + |> NaiveDateTime.truncate(:second) {first_datetime, last_datetime} end diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.ex index 225c0d8a5..c254df889 100644 --- a/lib/plausible/stats/current_visitors.ex +++ b/lib/plausible/stats/current_visitors.ex @@ -6,6 +6,7 @@ defmodule Plausible.Stats.CurrentVisitors do first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) + |> NaiveDateTime.truncate(:second) ClickhouseRepo.one( from e in "events", diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 32458037d..124320053 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -27,8 +27,8 @@ defmodule Plausible.Stats.Imported do from(s in Ecto.Query.subquery(native_q), full_join: i in subquery(imported_q), - on: field(s, :date) == field(i, :date), - select: %{date: field(s, :date)} + on: s.date == i.date, + select: %{date: fragment("greatest(?, ?)", s.date, i.date)} ) |> select_joined_metrics(metrics) end diff --git a/lib/plausible/stats/props.ex b/lib/plausible/stats/props.ex index 3e457f61c..7d4a26477 100644 --- a/lib/plausible/stats/props.ex +++ b/lib/plausible/stats/props.ex @@ -51,7 +51,7 @@ defmodule Plausible.Stats.Props do nil -> ClickhouseRepo.all( from e in base_event_query(site, query), - inner_lateral_join: meta in fragment("meta as m"), + inner_lateral_join: meta in fragment("meta"), select: {e.name, meta.key}, distinct: true ) diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 76fcec917..1125f3988 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -24,8 +24,9 @@ defmodule Plausible.Stats.Timeseries do Enum.map(steps, fn step -> empty_row(step, metrics) - |> Map.merge(Enum.find(event_result, fn row -> row[:date] == step end) || %{}) - |> Map.merge(Enum.find(session_result, fn row -> row[:date] == step end) || %{}) + |> Map.merge(Enum.find(event_result, fn row -> date_eq(row[:date], step) end) || %{}) + |> Map.merge(Enum.find(session_result, fn row -> date_eq(row[:date], step) end) || %{}) + |> Map.update!(:date, &date_format/1) end) end @@ -88,7 +89,6 @@ defmodule Plausible.Stats.Timeseries do query.date_range.first |> Timex.to_datetime() |> Timex.shift(hours: step) - |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") end) end @@ -109,10 +109,29 @@ defmodule Plausible.Stats.Timeseries do query.date_range.first |> Timex.to_datetime() |> Timex.shift(minutes: step) - |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}") end) end + defp date_eq(%DateTime{} = left, %DateTime{} = right) do + NaiveDateTime.compare(left, right) == :eq + end + + defp date_eq(%Date{} = left, %Date{} = right) do + Date.compare(left, right) == :eq + end + + defp date_eq(left, right) do + left == right + end + + defp date_format(%DateTime{} = date) do + Timex.format!(date, "{YYYY}-{0M}-{0D} {h24}:{m}:{s}") + end + + defp date_format(date) do + date + end + defp select_bucket(q, site, %Query{interval: "month"}) do from( e in q, diff --git a/mix.exs b/mix.exs index 50be46c35..c69c3f47f 100644 --- a/mix.exs +++ b/mix.exs @@ -62,8 +62,7 @@ defmodule Plausible.MixProject do {:bcrypt_elixir, "~> 2.0"}, {:bypass, "~> 2.1", only: [:dev, :test]}, {:cachex, "~> 3.4"}, - {:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"}, - {:clickhousex, github: "plausible/clickhousex", override: true}, + {:chto, github: "ruslandoga/chto"}, {:combination, "~> 0.0.3"}, {:connection, "~> 1.1", override: true}, {:cors_plug, "~> 3.0"}, diff --git a/mix.lock b/mix.lock index 979dc5c4c..ee8862deb 100644 --- a/mix.lock +++ b/mix.lock @@ -8,11 +8,11 @@ "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, - "castore": {:hex, :castore, "0.1.20", "62a0126cbb7cb3e259257827b9190f88316eb7aa3fdac01fd6f2dfd64e7f46e9", [:mix], [], "hexpm", "a020b7650529c986c454a4035b6b13a328e288466986307bea3aadb4c95ac98a"}, + "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, - "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "43126c020f07b097c55a81f68a906019fd061f29", []}, - "clickhousex": {:git, "https://github.com/plausible/clickhousex.git", "cfeefaf21f0841bc4e5adccb3b1875a1e5eb3ee7", []}, + "ch": {:git, "https://github.com/ruslandoga/ch.git", "5894f81e246c049b9db5186497189cce226947aa", []}, + "chatterbox": {:hex, :ts_chatterbox, "0.13.0", "6f059d97bcaa758b8ea6fffe2b3b81362bd06b639d3ea2bb088335511d691ebf", [:rebar3], [{:hpack, "~> 0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "b93d19104d86af0b3f2566c4cba2a57d2e06d103728246ba1ac6c3c0ff010aa7"}, + "chto": {:git, "https://github.com/ruslandoga/chto.git", "b5913b36331d5bb0a349b08b592da42410643cd9", []}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"}, @@ -32,7 +32,6 @@ "ecto": {:hex, :ecto, "3.9.4", "3ee68e25dbe0c36f980f1ba5dd41ee0d3eb0873bccae8aeaf1a2647242bffa35", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "de5f988c142a3aa4ec18b85a4ec34a2390b65b24f02385c1144252ff6ff8ee75"}, "ecto_sql": {:hex, :ecto_sql, "3.9.2", "34227501abe92dba10d9c3495ab6770e75e79b836d114c41108a4bf2ce200ad5", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1eb5eeb4358fdbcd42eac11c1fbd87e3affd7904e639d77903c1358b2abd3f70"}, "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, - "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, @@ -58,7 +57,6 @@ "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, - "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"}, diff --git a/priv/ingest_repo/migrations/.formatter.exs b/priv/ingest_repo/migrations/.formatter.exs new file mode 100644 index 000000000..49f9151ed --- /dev/null +++ b/priv/ingest_repo/migrations/.formatter.exs @@ -0,0 +1,4 @@ +[ + import_deps: [:ecto_sql], + inputs: ["*.exs"] +] diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs index 55fcb39ac..6f329cdfb 100644 --- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs +++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs @@ -8,8 +8,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do defp create_events() do create_if_not_exists table(:events, - engine: - "MergeTree() PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192" + primary_key: false, + engine: "MergeTree", + options: + "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192" ) do add(:name, :string) add(:domain, :string) @@ -30,15 +32,17 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do defp create_sessions() do create_if_not_exists table(:sessions, - engine: - "CollapsingMergeTree(sign) PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192" + primary_key: false, + engine: "CollapsingMergeTree(sign)", + options: + "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192" ) do add(:session_id, :UInt64) add(:sign, :Int8) add(:domain, :string) add(:user_id, :UInt64) add(:hostname, :string) - add(:is_bounce, :boolean) + add(:is_bounce, :UInt8) add(:entry_page, :string) add(:exit_page, :string) add(:pageviews, :integer) diff --git a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs index 32f2372df..4c1b6d44a 100644 --- a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs +++ b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEventMetadata do def change do alter table(:events) do - add :meta, {:nested, {{:key, :string}, {:value, :string}}} + add :meta, :"Nested(key String, value String)" end end end diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs index 33bc8b23c..5c7b72a87 100644 --- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs +++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs @@ -3,7 +3,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do def change do create_if_not_exists table(:imported_visitors, - engine: "MergeTree() ORDER BY (site_id, date)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -15,7 +17,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_sources, - engine: "MergeTree() ORDER BY (site_id, date, source)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, source)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -31,7 +35,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_pages, - engine: "MergeTree() ORDER BY (site_id, date, hostname, page)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, hostname, page)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -44,7 +50,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_entry_pages, - engine: "MergeTree() ORDER BY (site_id, date, entry_page)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, entry_page)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -56,7 +64,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_exit_pages, - engine: "MergeTree() ORDER BY (site_id, date, exit_page)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, exit_page)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -66,7 +76,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_locations, - engine: "MergeTree() ORDER BY (site_id, date, country, region, city)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, country, region, city)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -80,7 +92,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_devices, - engine: "MergeTree() ORDER BY (site_id, date, device)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, device)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -92,7 +106,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_browsers, - engine: "MergeTree() ORDER BY (site_id, date, browser)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, browser)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -104,7 +120,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_operating_systems, - engine: "MergeTree() ORDER BY (site_id, date, operating_system)" + primary_key: false, + engine: "MergeTree", + options: "ORDER BY (site_id, date, operating_system)" ) do add(:site_id, :UInt64) add(:date, :date) diff --git a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs index aa951a2d6..2c80ad8d6 100644 --- a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs +++ b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession do def change do alter table(:sessions) do - add :"entry.meta", {:nested, {{:key, :string}, {:value, :string}}} + add :"entry.meta", :"Nested(key String, value String)" end end end diff --git a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs index a2907a70a..ce45f6e3e 100644 --- a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs +++ b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps do def change do alter table(:sessions) do - add(:entry_meta, {:nested, {{:key, :string}, {:value, :string}}}) + add(:entry_meta, :"Nested(key String, value String)") end end end diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs index 231db40dc..75596571c 100644 --- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs +++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs @@ -3,8 +3,11 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do def change do create_if_not_exists table(:ingest_counters, - engine: "SummingMergeTree(value) ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))") do - + primary_key: false, + engine: "SummingMergeTree(value)", + options: + "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))" + ) do add(:event_timebucket, :utc_datetime) add(:domain, :"LowCardinality(String)") add(:site_id, :"Nullable(UInt64)") diff --git a/priv/ingest_repo/structure.sql b/priv/ingest_repo/structure.sql new file mode 100644 index 000000000..550aaa7d5 --- /dev/null +++ b/priv/ingest_repo/structure.sql @@ -0,0 +1,242 @@ +CREATE TABLE plausible_events_db.sessions +( + `session_id` UInt64, + `sign` Int8, + `domain` String, + `user_id` UInt64, + `hostname` String, + `is_bounce` UInt8, + `entry_page` String, + `exit_page` String, + `pageviews` Int32, + `events` Int32, + `duration` UInt32, + `referrer` String, + `referrer_source` String, + `country_code` LowCardinality(FixedString(2)), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `start` DateTime, + `timestamp` DateTime, + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String, + `entry_meta.key` Array(String), + `entry_meta.value` Array(String) +) +ENGINE = CollapsingMergeTree(sign) +PARTITION BY toYYYYMM(start) +ORDER BY (domain, toDate(start), user_id, session_id) +SAMPLE BY user_id +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.ingest_counters +( + `event_timebucket` DateTime, + `domain` LowCardinality(String), + `site_id` Nullable(UInt64), + `metric` LowCardinality(String), + `value` UInt64 +) +ENGINE = SummingMergeTree(value) +ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket)) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_visitors +( + `site_id` UInt64, + `date` Date, + `visitors` UInt64, + `pageviews` UInt64, + `bounces` UInt64, + `visits` UInt64, + `visit_duration` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_sources +( + `site_id` UInt64, + `date` Date, + `source` String, + `utm_medium` String, + `utm_campaign` String, + `utm_content` String, + `utm_term` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, source) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_pages +( + `site_id` UInt64, + `date` Date, + `hostname` String, + `page` String, + `visitors` UInt64, + `pageviews` UInt64, + `exits` UInt64, + `time_on_page` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, hostname, page) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_operating_systems +( + `site_id` UInt64, + `date` Date, + `operating_system` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, operating_system) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_locations +( + `site_id` UInt64, + `date` Date, + `country` String, + `region` String, + `city` UInt64, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, country, region, city) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_exit_pages +( + `site_id` UInt64, + `date` Date, + `exit_page` String, + `visitors` UInt64, + `exits` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, exit_page) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_entry_pages +( + `site_id` UInt64, + `date` Date, + `entry_page` String, + `visitors` UInt64, + `entrances` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, entry_page) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_devices +( + `site_id` UInt64, + `date` Date, + `device` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, device) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_browsers +( + `site_id` UInt64, + `date` Date, + `browser` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32 +) +ENGINE = MergeTree +ORDER BY (site_id, date, browser) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.events +( + `name` String, + `domain` String, + `user_id` UInt64, + `session_id` UInt64, + `hostname` String, + `pathname` String, + `referrer` String, + `referrer_source` String, + `country_code` LowCardinality(FixedString(2)), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `timestamp` DateTime, + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `meta.key` Array(String), + `meta.value` Array(String), + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(timestamp) +ORDER BY (domain, toDate(timestamp), user_id) +SAMPLE BY user_id +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.schema_migrations +( + `version` Int64, + `inserted_at` DateTime +) +ENGINE = TinyLog; + +INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES +(20200915070607,'2023-03-08 10:03:33'), +(20200918075025,'2023-03-08 10:03:33'), +(20201020083739,'2023-03-08 10:03:33'), +(20201106125234,'2023-03-08 10:03:33'), +(20210323130440,'2023-03-08 10:03:33'), +(20210712214034,'2023-03-08 10:03:33'), +(20211017093035,'2023-03-08 10:03:33'), +(20211112130238,'2023-03-08 10:03:33'), +(20220310104931,'2023-03-08 10:03:33'), +(20220404123000,'2023-03-08 10:03:33'), +(20220421161259,'2023-03-08 10:03:33'), +(20220422075510,'2023-03-08 10:03:33'), +(20230124140348,'2023-03-08 10:03:33'), +(20230210140348,'2023-03-08 10:03:33'), +(20230214114402,'2023-03-08 10:03:33'); diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 065749e1f..34fbf052c 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -7,7 +7,7 @@ defmodule Plausible.ImportedTest do defp import_data(ga_data, site_id, table_name) do ga_data |> Plausible.Imported.from_google_analytics(site_id, table_name) - |> then(&Plausible.IngestRepo.insert_all(table_name, &1)) + |> then(&Plausible.Google.Buffer.insert_all(table_name, &1)) end describe "Parse and import third party data fetched from Google Analytics" do diff --git a/test/plausible/ingestion/counters_test.exs b/test/plausible/ingestion/counters_test.exs index bf7d14c3e..19c73c9ce 100644 --- a/test/plausible/ingestion/counters_test.exs +++ b/test/plausible/ingestion/counters_test.exs @@ -37,8 +37,8 @@ defmodule Plausible.Ingestion.CountersTest do end test "the database eventually sums the records within 1-minute buckets", %{test: test} do - # Testing if the database works is an unfunny way of integration testing, - # but on the upside it's quite straight-forward way of testing if the + # Testing if the database works is an unfunny way of integration testing, + # but on the upside it's quite straight-forward way of testing if the # 1-minute bucket rollups are applied when dumping the records that are # originally aggregated with 10s windows. on_exit(:detach, fn -> diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 73f60fbad..5a719282a 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -39,6 +39,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert plot == [1] ++ zeroes ++ [1] end + test "displays visitors for a day with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00]), + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-01-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true" + ) + + assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200) + + assert plot == [2] ++ List.duplicate(0, 23) + end + test "displays hourly stats in configured timezone", %{conn: conn, user: user} do # UTC+1 site = @@ -60,7 +79,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert %{"plot" => plot} = json_response(conn, 200) - zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([]) + zeroes = List.duplicate(0, 22) # Expecting pageview to show at 1am CET assert plot == [0, 1] ++ zeroes @@ -108,6 +127,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 4 end + test "displays visitors for a month with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-01-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" + ) + + assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end + test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"), @@ -154,6 +193,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 4 end + test "displays visitors for 6 months with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-06-30]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 6 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end + test "displays visitors for 12 months with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), @@ -176,6 +235,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 4 end + test "displays visitors for 12 months with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-12-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=12mo&date=2021-12-31&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 12 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end + test "displays visitors for calendar year with imported data", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), @@ -198,6 +277,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 4 end + test "displays visitors for calendar year with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-12-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=year&date=2021-12-31&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 12 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end + test "displays visitors for all time with just native data", %{conn: conn, site: site} do use Plausible.Repo @@ -292,6 +391,26 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert List.last(plot) == 2 assert Enum.sum(plot) == 4 end + + test "displays pageviews for a month with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-01-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=pageviews&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end end describe "GET /api/stats/main-graph - bounce_rate plot" do @@ -337,6 +456,25 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert List.first(plot) == 50 assert List.last(plot) == 100 end + + test "displays bounce rate for a month with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, visits: 1, bounces: 0, date: ~D[2021-01-01]), + build(:imported_visitors, visits: 1, bounces: 1, date: ~D[2021-01-31]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=bounce_rate&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 0 + assert List.last(plot) == 100 + end end describe "GET /api/stats/main-graph - visit_duration plot" do @@ -387,6 +525,23 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.count(plot) == 31 assert List.first(plot) == 200 end + + test "displays visit_duration for a month with only imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:imported_visitors, visits: 1, visit_duration: 100, date: ~D[2021-01-01]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&metric=visit_duration&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 100 + end end describe "GET /api/stats/main-graph - varying intervals" do diff --git a/test/support/factory.ex b/test/support/factory.ex index 86f5d5719..629cdbb07 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -44,26 +44,12 @@ defmodule Plausible.Factory do user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), hostname: hostname, domain: hostname, - referrer: "", - referrer_source: "", - utm_medium: "", - utm_source: "", - utm_campaign: "", - utm_content: "", - utm_term: "", entry_page: "/", pageviews: 1, events: 1, - duration: 0, start: Timex.now(), timestamp: Timex.now(), - is_bounce: false, - browser: "", - browser_version: "", - country_code: "", - screen_size: "", - operating_system: "", - operating_system_version: "" + is_bounce: false } end @@ -85,22 +71,7 @@ defmodule Plausible.Factory do pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - referrer: "", - referrer_source: "", - utm_medium: "", - utm_source: "", - utm_campaign: "", - utm_content: "", - utm_term: "", - browser: "", - browser_version: "", - country_code: "", - screen_size: "", - operating_system: "", - operating_system_version: "", - "meta.key": [], - "meta.value": [] + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()) } end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index bb506edc5..a129fd2ad 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -73,28 +73,38 @@ defmodule Plausible.TestUtils do def create_pageviews(pageviews) do pageviews = Enum.map(pageviews, fn pageview -> - Factory.build(:pageview, pageview) |> Map.from_struct() |> Map.delete(:__meta__) + Factory.build(:pageview, pageview) + |> Map.from_struct() + |> Map.delete(:__meta__) + |> update_in([:timestamp], &to_naive_truncate/1) end) - Plausible.IngestRepo.insert_all("events", pageviews) + Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, pageviews) end def create_events(events) do events = Enum.map(events, fn event -> - Factory.build(:event, event) |> Map.from_struct() |> Map.delete(:__meta__) + Factory.build(:event, event) + |> Map.from_struct() + |> Map.delete(:__meta__) + |> update_in([:timestamp], &to_naive_truncate/1) end) - Plausible.IngestRepo.insert_all("events", events) + Plausible.IngestRepo.insert_all(Plausible.ClickhouseEvent, events) end def create_sessions(sessions) do sessions = Enum.map(sessions, fn session -> - Factory.build(:ch_session, session) |> Map.from_struct() |> Map.delete(:__meta__) + Factory.build(:ch_session, session) + |> Map.from_struct() + |> Map.delete(:__meta__) + |> update_in([:timestamp], &to_naive_truncate/1) + |> update_in([:start], &to_naive_truncate/1) end) - Plausible.IngestRepo.insert_all("sessions", sessions) + Plausible.IngestRepo.insert_all(Plausible.ClickhouseSession, sessions) end def log_in(%{user: user, conn: conn}) do @@ -136,7 +146,17 @@ defmodule Plausible.TestUtils do def populate_stats(events) do {native, imported} = - Enum.split_with(events, fn event -> + events + |> Enum.map(fn event -> + case event do + %{timestamp: timestamp} -> + %{event | timestamp: to_naive_truncate(timestamp)} + + _other -> + event + end + end) + |> Enum.split_with(fn event -> case event do %Plausible.ClickhouseEvent{} -> true @@ -168,7 +188,7 @@ defmodule Plausible.TestUtils do defp populate_imported_stats(events) do Enum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table)) - |> Enum.map(fn {table, events} -> Plausible.IngestRepo.insert_all(table, events) end) + |> Enum.map(fn {table, events} -> Plausible.Google.Buffer.insert_all(table, events) end) end def relative_time(shifts) do @@ -177,6 +197,14 @@ defmodule Plausible.TestUtils do |> NaiveDateTime.truncate(:second) end + def to_naive_truncate(%DateTime{} = dt) do + to_naive_truncate(DateTime.to_naive(dt)) + end + + def to_naive_truncate(%NaiveDateTime{} = naive) do + NaiveDateTime.truncate(naive, :second) + end + def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) do Enum.reduce_while(1..retries, nil, fn attempt, _acc -> case expectation.() do