From aa3a57a242602f1850965e8624bca4890af8b4c0 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 1 Mar 2023 11:18:12 +0200 Subject: [PATCH] WIP --- lib/plausible/stats/base.ex | 17 ++++++++++++----- lib/plausible/stats/clickhouse.ex | 4 +++- lib/plausible/stats/imported.ex | 10 +++++----- lib/plausible/stats/timeseries.ex | 3 ++- test/support/factory.ex | 3 ++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 9e42c536a..21a098b1d 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base do end def query_events(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + {first_datetime, last_datetime} = utc_boundaries(query, site) q = from( @@ -145,7 +145,7 @@ defmodule Plausible.Stats.Base do } def query_sessions(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + {first_datetime, last_datetime} = utc_boundaries(query, site) sessions_q = from( @@ -409,17 +409,24 @@ defmodule Plausible.Stats.Base do {first_datetime, last_datetime} end - def utc_boundaries(%Query{date_range: date_range}, timezone) do + def utc_boundaries(%Query{date_range: date_range}, site) do {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) first_datetime = - Timex.to_datetime(first, timezone) + Timex.to_datetime(first, site.timezone) |> Timex.Timezone.convert("UTC") + first_datetime = + if Timex.after?(site.inserted_at, first_datetime) do + site.inserted_at + else + first_datetime + end + {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) last_datetime = - Timex.to_datetime(last, timezone) + Timex.to_datetime(last, site.timezone) |> Timex.Timezone.convert("UTC") {first_datetime, last_datetime} diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 32077ebb4..5fd6823e7 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -7,6 +7,7 @@ defmodule Plausible.Stats.Clickhouse do @spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil def pageview_start_date_local(site) do + # TODO: limit by site.inserted_at datetime = ClickhouseRepo.one( from e in "events", @@ -46,7 +47,7 @@ defmodule Plausible.Stats.Clickhouse do def usage_breakdown(domains, date_range) do Enum.chunk_every(domains, 300) - |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} -> + |> Enum.reduce({0, 0}, fn domains, j({pageviews_total, custom_events_total}) -> {chunk_pageviews, chunk_custom_events} = ClickhouseRepo.one( from e in "events", @@ -155,6 +156,7 @@ defmodule Plausible.Stats.Clickhouse do ) end + # TODO: Needs limit by site.inserted_at def has_pageviews?([]), do: false def has_pageviews?(domains) when is_list(domains) do diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 3788f734f..9f693f0f7 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -23,7 +23,7 @@ defmodule Plausible.Stats.Imported do select: %{} ) |> select_imported_metrics(metrics) - |> apply_interval(query, site.timezone) + |> apply_interval(query, site) from(s in Ecto.Query.subquery(native_q), full_join: i in subquery(imported_q), @@ -33,21 +33,21 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(metrics) end - defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _timezone) do + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}, _site) do imported_q |> group_by([i], fragment("toStartOfMonth(?)", i.date)) |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)}) end - defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, timezone) do - {first_datetime, _} = utc_boundaries(query, timezone) + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "week"} = query, site) do + {first_datetime, _} = utc_boundaries(query, site) imported_q |> group_by([i], weekstart_not_before(i.date, ^first_datetime)) |> select_merge([i], %{date: weekstart_not_before(i.date, ^first_datetime)}) end - defp apply_interval(imported_q, _query, _timezone) do + defp apply_interval(imported_q, _query, _site) do imported_q |> group_by([i], i.date) |> select_merge([i], %{date: i.date}) diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 37ca49aa5..165dc7497 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -121,7 +121,8 @@ defmodule Plausible.Stats.Timeseries do end defp select_bucket(q, site, %Query{interval: "week"} = query) do - {first_datetime, _} = utc_boundaries(query, site.timezone) + # TODO: Maybe use query.date_range.first + {first_datetime, _} = utc_boundaries(query, site) from( e in q, diff --git a/test/support/factory.ex b/test/support/factory.ex index 9797fb2fb..524bdb64d 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -26,7 +26,8 @@ defmodule Plausible.Factory do %Plausible.Site{ domain: domain, - timezone: "UTC" + timezone: "UTC", + inserted_at: ~N[2020-01-01 00:00:00] } end