diff --git a/assets/js/dashboard/site-switcher.js b/assets/js/dashboard/site-switcher.js index d18a4abed..476ccf7e6 100644 --- a/assets/js/dashboard/site-switcher.js +++ b/assets/js/dashboard/site-switcher.js @@ -90,8 +90,8 @@ export default class SiteSwitcher extends React.Component {
- - Site settings + + Site settings
diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 85c75b5e6..2bae9afca 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -285,11 +285,25 @@ class LineGraph extends React.Component { return ( - + ) } + samplingNotice() { + const samplePercent = this.props.graphData.sample_percent + + if (samplePercent < 100) { + return ( +
+ + + +
+ ) + } + } + render() { const extraClass = this.props.graphData.interval === 'hour' ? '' : 'cursor-pointer' @@ -300,6 +314,7 @@ class LineGraph extends React.Component {
{ this.downloadLink() } + { this.samplingNotice() }
@@ -349,7 +364,7 @@ export default class VisitorGraph extends React.Component { render() { return ( -
+
{ this.state.loading &&
} { this.renderInner() }
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 29e9d9785..aebe43bcd 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -2,6 +2,7 @@ defmodule Plausible.Stats.Clickhouse do use Plausible.Repo use Plausible.ClickhouseRepo alias Plausible.Stats.Query + use Plausible.Stats.Fragments @no_ref "Direct / None" def compare_pageviews_and_visitors(site, query, {pageviews, visitors}) do @@ -39,7 +40,7 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), select: {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone), - fragment("uniq(?)", e.user_id)}, + uniq(e.user_id)}, group_by: fragment("month"), order_by: fragment("month") ) @@ -64,7 +65,7 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), select: {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone), - fragment("uniq(?)", e.user_id)}, + uniq(e.user_id)}, group_by: fragment("day"), order_by: fragment("day") ) @@ -88,7 +89,7 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), select: {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone), - fragment("uniq(?)", e.user_id)}, + uniq(e.user_id)}, group_by: fragment("hour"), order_by: fragment("hour") ) @@ -118,7 +119,7 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), select: { fragment("dateDiff('minute', now(), ?) as relativeMinute", e.timestamp), - fragment("count(*)") + total() }, group_by: fragment("relativeMinute"), order_by: fragment("relativeMinute") @@ -135,7 +136,7 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.one( from s in q, - select: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)") + select: bounce_rate() ) || 0 end @@ -144,7 +145,7 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.one( from s in q, - select: fragment("round(avg(duration * sign))") + select: visit_duration() ) || 0 end @@ -162,14 +163,14 @@ defmodule Plausible.Stats.Clickhouse do def total_pageviews(site, query) do ClickhouseRepo.one( from e in base_query_w_sessions(site, query), - select: fragment("count(*)") + select: total() ) end def total_events(site, query) do ClickhouseRepo.one( from e in base_query_w_sessions(site, query), - select: fragment("count(*) as events") + select: total() ) end @@ -191,18 +192,28 @@ defmodule Plausible.Stats.Clickhouse do end def pageviews_and_visitors(site, query) do + {pageviews, visitors, _} = pageviews_and_visitors_with_sample_percent(site, query) + {pageviews, visitors} + end + + def pageviews_and_visitors_with_sample_percent(site, query) do ClickhouseRepo.one( from e in base_query_w_sessions(site, query), - select: {fragment("count(*)"), fragment("uniq(user_id)")} + select: {total(), uniq(e.user_id), sample_percent()} ) end def unique_visitors(site, query) do + {visitors, _} = unique_visitors_with_sample_percent(site, query) + visitors + end + + def unique_visitors_with_sample_percent(site, query) do query = if query.period == "realtime", do: %Query{query | period: "30m"}, else: query ClickhouseRepo.one( from e in base_query_w_sessions(site, query), - select: fragment("uniq(user_id)") + select: {uniq(e.user_id), sample_percent()} ) end @@ -213,13 +224,13 @@ defmodule Plausible.Stats.Clickhouse do from s in base_query_w_sessions(site, query), where: s.referrer_source != "", group_by: s.referrer_source, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: ^limit, offset: ^offset, select: %{ name: s.referrer_source, url: fragment("any(?)", s.referrer), - count: fragment("uniq(?) as count", s.user_id) + count: uniq(s.user_id) } ) |> Enum.map(fn ref -> @@ -233,7 +244,7 @@ defmodule Plausible.Stats.Clickhouse do referrers = from(s in base_session_query(site, query), group_by: s.referrer_source, - order_by: [desc: fragment("count"), asc: fragment("min(start)")], + order_by: [desc: uniq(s.user_id), asc: fragment("min(start)")], limit: ^limit, offset: ^offset ) @@ -267,10 +278,9 @@ defmodule Plausible.Stats.Clickhouse do s.referrer_source ), url: fragment("any(?)", s.referrer), - count: fragment("uniq(user_id) as count"), - bounce_rate: - fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate"), - visit_duration: fragment("round(avg(duration * sign)) as visit_duration") + count: uniq(s.user_id), + bounce_rate: bounce_rate(), + visit_duration: visit_duration() } ) else @@ -285,7 +295,7 @@ defmodule Plausible.Stats.Clickhouse do s.referrer_source ), url: fragment("any(?)", s.referrer), - count: fragment("uniq(user_id) as count") + count: uniq(s.user_id) } ) end @@ -332,14 +342,14 @@ defmodule Plausible.Stats.Clickhouse do from( s in base_session_query(site, query), group_by: s.utm_medium, - order_by: [desc: fragment("count"), asc: fragment("min(start)")], + order_by: [desc: uniq(s.user_id), asc: min(s.start)], limit: ^limit, offset: ^offset, select: %{ - name: fragment("if(empty(?), ?, ?) as name", s.utm_medium, @no_ref, s.utm_medium), - count: fragment("uniq(user_id) as count"), - bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), - visit_duration: fragment("round(avg(duration * sign))") + name: coalesce_string(s.utm_medium, @no_ref), + count: uniq(s.user_id), + bounce_rate: bounce_rate(), + visit_duration: visit_duration() } ) @@ -362,14 +372,14 @@ defmodule Plausible.Stats.Clickhouse do from( s in base_session_query(site, query), group_by: s.utm_campaign, - order_by: [desc: fragment("count"), asc: fragment("min(start)")], + order_by: [desc: uniq(s.user_id), asc: min(s.start)], limit: ^limit, offset: ^offset, select: %{ - name: fragment("if(empty(?), ?, ?) as name", s.utm_campaign, @no_ref, s.utm_campaign), - count: fragment("uniq(user_id) as count"), - bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), - visit_duration: fragment("round(avg(duration * sign))") + name: coalesce_string(s.utm_campaign, @no_ref), + count: uniq(s.user_id), + bounce_rate: bounce_rate(), + visit_duration: visit_duration() } ) @@ -392,14 +402,14 @@ defmodule Plausible.Stats.Clickhouse do from( s in base_session_query(site, query), group_by: s.utm_source, - order_by: [desc: fragment("count"), asc: fragment("min(start)")], + order_by: [desc: uniq(s.user_id), asc: min(s.start)], limit: ^limit, offset: ^offset, select: %{ - name: fragment("if(empty(?), ?, ?) as name", s.utm_source, @no_ref, s.utm_source), - count: fragment("uniq(user_id) as count"), - bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"), - visit_duration: fragment("round(avg(duration * sign))") + name: coalesce_string(s.utm_source, @no_ref), + count: uniq(s.user_id), + bounce_rate: bounce_rate(), + visit_duration: visit_duration() } ) @@ -427,7 +437,7 @@ defmodule Plausible.Stats.Clickhouse do join: cs in subquery(converted_sessions), on: s.session_id == cs.session_id, where: s.referrer_source == ^referrer, - select: fragment("uniq(user_id) as visitors") + select: uniq(s.user_id) ) end @@ -439,7 +449,7 @@ defmodule Plausible.Stats.Clickhouse do s in base_session_query(site, query), group_by: s.referrer, where: s.referrer_source == ^referrer, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: ^limit ) |> filter_converted_sessions(site, query) @@ -449,18 +459,17 @@ defmodule Plausible.Stats.Clickhouse do from( s in q, select: %{ - name: fragment("if(empty(?), ?, ?) as name", s.referrer, @no_ref, s.referrer), - count: fragment("uniq(user_id) as count"), - bounce_rate: - fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate"), - visit_duration: fragment("round(avg(duration * sign)) as visit_duration") + name: coalesce_string(s.referrer, @no_ref), + count: uniq(s.user_id), + bounce_rate: bounce_rate(), + visit_duration: visit_duration() } ) else from(s in q, select: %{ - name: fragment("if(empty(?), ?, ?) as name", s.referrer, @no_ref, s.referrer), - count: fragment("uniq(user_id) as count") + name: coalesce_string(s.referrer, @no_ref), + count: uniq(s.user_id) } ) end @@ -495,11 +504,11 @@ defmodule Plausible.Stats.Clickhouse do from s in base_query_w_sessions(site, query), where: s.referrer_source == ^referrer, group_by: s.referrer, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: 100, select: %{ name: s.referrer, - count: fragment("uniq(user_id) as count") + count: uniq(s.user_id) } ) end @@ -511,14 +520,14 @@ defmodule Plausible.Stats.Clickhouse do from( s in base_session_query(site, query), group_by: s.entry_page, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: ^limit, offset: ^offset, select: %{ name: s.entry_page, - count: fragment("uniq(?) as count", s.user_id), - entries: fragment("uniq(?) as entries", s.session_id), - visit_duration: fragment("avg(?) as visit_duration", s.duration) + count: uniq(s.user_id), + entries: uniq(s.session_id), + visit_duration: visit_duration() } ) |> filter_converted_sessions(site, query) @@ -533,14 +542,14 @@ defmodule Plausible.Stats.Clickhouse do from( s in base_session_query(site, query), group_by: s.exit_page, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: ^limit, offset: ^offset, where: s.exit_page != "", select: %{ name: s.exit_page, - count: fragment("uniq(?) as count", s.user_id), - exits: fragment("uniq(?) as exits", s.session_id) + count: uniq(s.user_id), + exits: uniq(s.session_id) } ) |> filter_converted_sessions(site, query) @@ -557,7 +566,7 @@ defmodule Plausible.Stats.Clickhouse do where: e.name == "pageview", select: { e.pathname, - fragment("count(*)") + total() } ) @@ -584,12 +593,12 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.all( from s in q, group_by: s.exit_page, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(s.user_id)], limit: ^limit, offset: ^offset, select: %{ - name: fragment("? as name", s.exit_page), - count: fragment("uniq(?) as count", s.user_id) + name: s.exit_page, + count: uniq(s.user_id) } ) end @@ -601,13 +610,13 @@ defmodule Plausible.Stats.Clickhouse do from( e in base_query_w_sessions(site, query), group_by: e.pathname, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], limit: ^limit, offset: ^offset, select: %{ - name: fragment("? as name", e.pathname), - count: fragment("uniq(?) as count", e.user_id), - pageviews: fragment("count(*) as pageviews") + name: e.pathname, + count: uniq(e.user_id), + pageviews: total() } ) @@ -627,12 +636,12 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.all( from s in q, group_by: s.entry_page, - order_by: [desc: fragment("total")], + order_by: [desc: total()], limit: 100, select: %{ entry_page: s.entry_page, - total: fragment("count(*) as total"), - bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate") + total: total(), + bounce_rate: bounce_rate() } ) |> Enum.map(fn row -> {row[:entry_page], row[:bounce_rate]} end) @@ -652,10 +661,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.screen_size, where: e.screen_size != "", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.screen_size, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> add_percentages @@ -666,10 +675,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.country_code, where: e.country_code != "\0\0", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.country_code, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> Enum.map(fn stat -> @@ -687,10 +696,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.browser, where: e.browser != "", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.browser, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> add_percentages @@ -702,10 +711,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.browser_version, where: e.browser_version != "", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.browser_version, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> add_percentages @@ -717,10 +726,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.operating_system, where: e.operating_system != "", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.operating_system, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> add_percentages @@ -732,10 +741,10 @@ defmodule Plausible.Stats.Clickhouse do from e in base_query_w_sessions(site, query), group_by: e.operating_system_version, where: e.operating_system_version != "", - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: e.operating_system_version, - count: fragment("uniq(user_id) as count") + count: uniq(e.user_id) } ) |> add_percentages @@ -744,8 +753,8 @@ defmodule Plausible.Stats.Clickhouse do def current_visitors(site, query) do Plausible.ClickhouseRepo.one( - from s in base_query(site, query), - select: fragment("uniq(user_id)") + from e in base_query(site, query), + select: uniq(e.user_id) ) end @@ -801,22 +810,22 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.all( from e in base_query_w_sessions(site, query), where: fragment("not has(meta.key, ?)", ^key), - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: "(none)", - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) else ClickhouseRepo.all( from [e, meta: meta] in base_query_w_sessions(site, query), group_by: meta.value, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: meta.value, - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) end @@ -829,8 +838,8 @@ defmodule Plausible.Stats.Clickhouse do where: fragment("not has(?.key, ?)", e.meta, ^key), select: %{ name: "(none)", - count: fragment("uniq(?) as count", e.user_id), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) @@ -840,11 +849,11 @@ defmodule Plausible.Stats.Clickhouse do inner_lateral_join: meta in fragment("meta as m"), where: meta.key == ^key, group_by: meta.value, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: meta.value, - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) @@ -881,11 +890,11 @@ defmodule Plausible.Stats.Clickhouse do ClickhouseRepo.all( from e in base_query_w_sessions(site, query), group_by: e.name, - order_by: [desc: fragment("count")], + order_by: [desc: uniq(e.user_id)], select: %{ name: ^goal, - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) end @@ -912,8 +921,8 @@ defmodule Plausible.Stats.Clickhouse do group_by: e.name, select: %{ name: e.name, - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() } ) @@ -946,8 +955,8 @@ defmodule Plausible.Stats.Clickhouse do ), select: %{ index: fragment("arrayJoin(indices) as index"), - count: fragment("uniq(user_id) as count"), - total_count: fragment("count(*) as total_count") + count: uniq(e.user_id), + total_count: total() }, group_by: fragment("index") ) @@ -969,6 +978,7 @@ defmodule Plausible.Stats.Clickhouse do sessions_q = from(s in "sessions", + hints: ["SAMPLE 10000000"], where: s.domain == ^site.domain, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime, select: %{session_id: s.session_id} @@ -1081,6 +1091,7 @@ defmodule Plausible.Stats.Clickhouse do q = from(e in "events", + hints: ["SAMPLE 10000000"], where: e.domain == ^site.domain, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) @@ -1141,6 +1152,7 @@ defmodule Plausible.Stats.Clickhouse do q = from(e in "events", + hints: ["SAMPLE 10000000"], where: e.domain == ^site.domain, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) @@ -1169,6 +1181,7 @@ defmodule Plausible.Stats.Clickhouse do q = from(s in "sessions", + hints: ["SAMPLE 10000000"], where: s.domain == ^site.domain, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime ) @@ -1283,6 +1296,7 @@ defmodule Plausible.Stats.Clickhouse do q = from(e in "events", + hints: ["SAMPLE 10000000"], where: e.domain == ^site.domain, where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime ) diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex new file mode 100644 index 000000000..afa9c8f6d --- /dev/null +++ b/lib/plausible/stats/fragments.ex @@ -0,0 +1,43 @@ +defmodule Plausible.Stats.Fragments do + defmacro uniq(user_id) do + quote do + fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id)) + end + end + + defmacro total() do + quote do + fragment("toUInt64(round(count(*) * any(_sample_factor)))") + end + end + + defmacro sample_percent() do + quote do + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + end + end + + defmacro bounce_rate() do + quote do + fragment("toUInt64(round(sum(is_bounce * sign) / sum(sign) * 100))") + end + end + + defmacro visit_duration() do + quote do + fragment("toUInt64(round(avg(duration * sign)))") + end + end + + defmacro coalesce_string(fieldA, fieldB) do + quote do + fragment("if(empty(?), ?, ?)", unquote(fieldA), unquote(fieldB), unquote(fieldA)) + end + end + + defmacro __using__(_) do + quote do + import Plausible.Stats.Fragments + end + end +end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 1dad18eaf..28ee9f324 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.StatsController do query = Query.from(site.timezone, params) plot_task = Task.async(fn -> Stats.calculate_plot(site, query) end) - top_stats = fetch_top_stats(site, query) + {top_stats, sample_percent} = fetch_top_stats(site, query) {plot, labels, present_index} = Task.await(plot_task) json(conn, %{ @@ -17,12 +17,13 @@ defmodule PlausibleWeb.Api.StatsController do labels: labels, present_index: present_index, top_stats: top_stats, - interval: query.interval + interval: query.interval, + sample_percent: sample_percent }) end defp fetch_top_stats(site, %Query{period: "realtime"} = query) do - [ + stats = [ %{ name: "Current visitors", count: Stats.current_visitors(site, query) @@ -36,6 +37,8 @@ defmodule PlausibleWeb.Api.StatsController do count: Stats.total_pageviews(site, query) } ] + + {stats, 100} end defp fetch_top_stats(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do @@ -43,7 +46,7 @@ defmodule PlausibleWeb.Api.StatsController do prev_query = Query.shift_back(query, site) unique_visitors = Stats.unique_visitors(site, %{query | filters: total_filter}) prev_unique_visitors = Stats.unique_visitors(site, %{prev_query | filters: total_filter}) - converted_visitors = Stats.unique_visitors(site, query) + {converted_visitors, sample_percent} = Stats.unique_visitors_with_sample_percent(site, query) prev_converted_visitors = Stats.unique_visitors(site, prev_query) completions = Stats.total_events(site, query) prev_completions = Stats.total_events(site, prev_query) @@ -51,7 +54,7 @@ defmodule PlausibleWeb.Api.StatsController do conversion_rate = calculate_cr(unique_visitors, converted_visitors) prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors) - [ + stats = [ %{ name: "Unique visitors", count: unique_visitors, @@ -73,11 +76,16 @@ defmodule PlausibleWeb.Api.StatsController do change: percent_change(prev_conversion_rate, conversion_rate) } ] + + {stats, sample_percent} end defp fetch_top_stats(site, query) do prev_query = Query.shift_back(query, site) - {pageviews, visitors} = Stats.pageviews_and_visitors(site, query) + + {pageviews, visitors, sample_percent} = + Stats.pageviews_and_visitors_with_sample_percent(site, query) + {prev_pageviews, prev_visitors} = Stats.pageviews_and_visitors(site, prev_query) bounce_rate = Stats.bounce_rate(site, query) prev_bounce_rate = Stats.bounce_rate(site, prev_query) @@ -95,21 +103,24 @@ defmodule PlausibleWeb.Api.StatsController do } end - [ - %{ - name: "Unique visitors", - count: visitors, - change: percent_change(prev_visitors, visitors) - }, - %{ - name: "Total pageviews", - count: pageviews, - change: percent_change(prev_pageviews, pageviews) - }, - %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate}, - visit_duration - ] - |> Enum.filter(& &1) + stats = + [ + %{ + name: "Unique visitors", + count: visitors, + change: percent_change(prev_visitors, visitors) + }, + %{ + name: "Total pageviews", + count: pageviews, + change: percent_change(prev_pageviews, pageviews) + }, + %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate}, + visit_duration + ] + |> Enum.filter(& &1) + + {stats, sample_percent} end defp percent_change(old_count, new_count) do diff --git a/mix.lock b/mix.lock index a2f4b80e7..9bc10cdd0 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, "cachex": {:hex, :cachex, "3.3.0", "6f2ebb8f27491fe39121bd207c78badc499214d76c695658b19d6079beeca5c2", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "d90e5ee1dde14cef33f6b187af4335b88748b72b30c038969176cd4e6ccc31a1"}, "certifi": {:hex, :certifi, "2.5.3", "70bdd7e7188c804f3a30ee0e7c99655bc35d8ac41c23e12325f36ab449b70651", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "ed516acb3929b101208a9d700062d520f3953da3b6b918d866106ffa980e1c10"}, - "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "918e84e55072948170b5c8a2207ba394aaeb8e5b", []}, + "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "b30ccc93a4101a25ff0bba92113e18d8a9a8b28e", []}, "clickhousex": {:git, "https://github.com/plausible/clickhousex", "0832dd4b1af1f0eba1d1018c231bf0d8d281f031", []}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"}, diff --git a/priv/clickhouse_repo/migrations/20210323130440_add_sample_by.exs b/priv/clickhouse_repo/migrations/20210323130440_add_sample_by.exs new file mode 100644 index 000000000..1a547ad95 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20210323130440_add_sample_by.exs @@ -0,0 +1,8 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddSampleBy do + use Ecto.Migration + + def change do + execute "ALTER TABLE events MODIFY SAMPLE BY user_id" + execute "ALTER TABLE sessions MODIFY SAMPLE BY user_id" + end +end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index fef444303..d431b7c66 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -97,14 +97,14 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01") res = json_response(conn, 200) - assert %{"name" => "Bounce rate", "percentage" => 33.0, "change" => nil} in res["top_stats"] + assert %{"name" => "Bounce rate", "percentage" => 33, "change" => nil} in res["top_stats"] end test "calculates average visit duration", %{conn: conn, site: site} do conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01") res = json_response(conn, 200) - assert %{"name" => "Visit duration", "count" => 67.0, "change" => 100} in res["top_stats"] + assert %{"name" => "Visit duration", "count" => 67, "change" => 100} in res["top_stats"] end end diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index e6fcf1121..69794fb00 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -72,7 +72,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do "count" => 3, "entries" => 3, "name" => "/", - "visit_duration" => 66.66666666666667 + "visit_duration" => 67 } ] end @@ -86,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do assert json_response(conn, 200) == [ %{ - "visit_duration" => 66.66666666666667, + "visit_duration" => 67, "count" => 3, "entries" => 3, "name" => "/"