Add support for sampling (#875)

* Add support for sampling

* Use correct mix package

* Add notice about data sampling
pull/859/head
Uku Taht 6 years ago committed by GitHub
parent 96633f46b1
commit 3eb2f6d7a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -90,8 +90,8 @@ export default class SiteSwitcher extends React.Component {
<React.Fragment>
<div className="py-1">
<a href={`/${encodeURIComponent(this.props.site.domain)}/settings`} className="group flex items-center px-4 py-2 md:text-sm leading-5 text-gray-700 dark:text-gray-300 hover:bg-gray-100 dark:hover:bg-gray-900 hover:text-gray-900 dark:hover:text-gray-100 focus:outline-none focus:bg-gray-100 dark:focus:bg-gray-900 focus:text-gray-900 dark:focus:text-gray-100" role="menuitem">
<svg viewBox="0 0 20 20" fill="currentColor" className="mr-2 h-4 w-4 text-gray-500 dark:text-gray-200 group-hover:text-gray-600 dark:group-hover:text-gray-400 group-focus:text-gray-500 dark:group-focus:text-gray-200"><path d="M5 4a1 1 0 00-2 0v7.268a2 2 0 000 3.464V16a1 1 0 102 0v-1.268a2 2 0 000-3.464V4zM11 4a1 1 0 10-2 0v1.268a2 2 0 000 3.464V16a1 1 0 102 0V8.732a2 2 0 000-3.464V4zM16 3a1 1 0 011 1v7.268a2 2 0 010 3.464V16a1 1 0 11-2 0v-1.268a2 2 0 010-3.464V4a1 1 0 011-1z" /></svg>
Site settings
<svg class="mr-2 h-4 w-4 text-gray-500 dark:text-gray-200 group-hover:text-gray-600 dark:group-hover:text-gray-400 group-focus:text-gray-500 dark:group-focus:text-gray-200" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" d="M11.49 3.17c-.38-1.56-2.6-1.56-2.98 0a1.532 1.532 0 01-2.286.948c-1.372-.836-2.942.734-2.106 2.106.54.886.061 2.042-.947 2.287-1.561.379-1.561 2.6 0 2.978a1.532 1.532 0 01.947 2.287c-.836 1.372.734 2.942 2.106 2.106a1.532 1.532 0 012.287.947c.379 1.561 2.6 1.561 2.978 0a1.533 1.533 0 012.287-.947c1.372.836 2.942-.734 2.106-2.106a1.533 1.533 0 01.947-2.287c1.561-.379 1.561-2.6 0-2.978a1.532 1.532 0 01-.947-2.287c.836-1.372-.734-2.942-2.106-2.106a1.532 1.532 0 01-2.287-.947zM10 13a3 3 0 100-6 3 3 0 000 6z" clip-rule="evenodd"></path></svg>
Site settings
</a>
</div>
<div className="border-t border-gray-200 dark:border-gray-500"></div>

@ -285,11 +285,25 @@ class LineGraph extends React.Component {
return (
<a href={endpoint} download>
<svg className="absolute w-4 h-5 text-gray-700 feather dark:text-gray-300" style={{right: '2rem', top: '-2rem'}} xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
<svg className="absolute w-4 h-5 text-gray-700 feather dark:text-gray-300 -top-8 right-8" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
</a>
)
}
samplingNotice() {
const samplePercent = this.props.graphData.sample_percent
if (samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="absolute cursor-pointer -top-20 right-8">
<svg className="w-4 h-4 text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
}
}
render() {
const extraClass = this.props.graphData.interval === 'hour' ? '' : 'cursor-pointer'
@ -300,6 +314,7 @@ class LineGraph extends React.Component {
</div>
<div className="relative px-2">
{ this.downloadLink() }
{ this.samplingNotice() }
<canvas id="main-graph-canvas" className={'mt-4 ' + extraClass} width="1054" height="342"></canvas>
</div>
</div>
@ -349,7 +364,7 @@ export default class VisitorGraph extends React.Component {
render() {
return (
<div className="relative w-full mt-6 bg-white rounded shadow-xl dark:bg-gray-825 main-graph">
<div className="relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825 main-graph">
{ this.state.loading && <div className="graph-inner"><div className="pt-24 mx-auto loading sm:pt-32 md:pt-48"><div></div></div></div> }
{ this.renderInner() }
</div>

@ -2,6 +2,7 @@ defmodule Plausible.Stats.Clickhouse do
use Plausible.Repo
use Plausible.ClickhouseRepo
alias Plausible.Stats.Query
use Plausible.Stats.Fragments
@no_ref "Direct / None"
def compare_pageviews_and_visitors(site, query, {pageviews, visitors}) do
@ -39,7 +40,7 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
select:
{fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone),
fragment("uniq(?)", e.user_id)},
uniq(e.user_id)},
group_by: fragment("month"),
order_by: fragment("month")
)
@ -64,7 +65,7 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
select:
{fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone),
fragment("uniq(?)", e.user_id)},
uniq(e.user_id)},
group_by: fragment("day"),
order_by: fragment("day")
)
@ -88,7 +89,7 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
select:
{fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone),
fragment("uniq(?)", e.user_id)},
uniq(e.user_id)},
group_by: fragment("hour"),
order_by: fragment("hour")
)
@ -118,7 +119,7 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
select: {
fragment("dateDiff('minute', now(), ?) as relativeMinute", e.timestamp),
fragment("count(*)")
total()
},
group_by: fragment("relativeMinute"),
order_by: fragment("relativeMinute")
@ -135,7 +136,7 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.one(
from s in q,
select: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)")
select: bounce_rate()
) || 0
end
@ -144,7 +145,7 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.one(
from s in q,
select: fragment("round(avg(duration * sign))")
select: visit_duration()
) || 0
end
@ -162,14 +163,14 @@ defmodule Plausible.Stats.Clickhouse do
def total_pageviews(site, query) do
ClickhouseRepo.one(
from e in base_query_w_sessions(site, query),
select: fragment("count(*)")
select: total()
)
end
def total_events(site, query) do
ClickhouseRepo.one(
from e in base_query_w_sessions(site, query),
select: fragment("count(*) as events")
select: total()
)
end
@ -191,18 +192,28 @@ defmodule Plausible.Stats.Clickhouse do
end
def pageviews_and_visitors(site, query) do
{pageviews, visitors, _} = pageviews_and_visitors_with_sample_percent(site, query)
{pageviews, visitors}
end
def pageviews_and_visitors_with_sample_percent(site, query) do
ClickhouseRepo.one(
from e in base_query_w_sessions(site, query),
select: {fragment("count(*)"), fragment("uniq(user_id)")}
select: {total(), uniq(e.user_id), sample_percent()}
)
end
def unique_visitors(site, query) do
{visitors, _} = unique_visitors_with_sample_percent(site, query)
visitors
end
def unique_visitors_with_sample_percent(site, query) do
query = if query.period == "realtime", do: %Query{query | period: "30m"}, else: query
ClickhouseRepo.one(
from e in base_query_w_sessions(site, query),
select: fragment("uniq(user_id)")
select: {uniq(e.user_id), sample_percent()}
)
end
@ -213,13 +224,13 @@ defmodule Plausible.Stats.Clickhouse do
from s in base_query_w_sessions(site, query),
where: s.referrer_source != "",
group_by: s.referrer_source,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: ^limit,
offset: ^offset,
select: %{
name: s.referrer_source,
url: fragment("any(?)", s.referrer),
count: fragment("uniq(?) as count", s.user_id)
count: uniq(s.user_id)
}
)
|> Enum.map(fn ref ->
@ -233,7 +244,7 @@ defmodule Plausible.Stats.Clickhouse do
referrers =
from(s in base_session_query(site, query),
group_by: s.referrer_source,
order_by: [desc: fragment("count"), asc: fragment("min(start)")],
order_by: [desc: uniq(s.user_id), asc: fragment("min(start)")],
limit: ^limit,
offset: ^offset
)
@ -267,10 +278,9 @@ defmodule Plausible.Stats.Clickhouse do
s.referrer_source
),
url: fragment("any(?)", s.referrer),
count: fragment("uniq(user_id) as count"),
bounce_rate:
fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate"),
visit_duration: fragment("round(avg(duration * sign)) as visit_duration")
count: uniq(s.user_id),
bounce_rate: bounce_rate(),
visit_duration: visit_duration()
}
)
else
@ -285,7 +295,7 @@ defmodule Plausible.Stats.Clickhouse do
s.referrer_source
),
url: fragment("any(?)", s.referrer),
count: fragment("uniq(user_id) as count")
count: uniq(s.user_id)
}
)
end
@ -332,14 +342,14 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in base_session_query(site, query),
group_by: s.utm_medium,
order_by: [desc: fragment("count"), asc: fragment("min(start)")],
order_by: [desc: uniq(s.user_id), asc: min(s.start)],
limit: ^limit,
offset: ^offset,
select: %{
name: fragment("if(empty(?), ?, ?) as name", s.utm_medium, @no_ref, s.utm_medium),
count: fragment("uniq(user_id) as count"),
bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"),
visit_duration: fragment("round(avg(duration * sign))")
name: coalesce_string(s.utm_medium, @no_ref),
count: uniq(s.user_id),
bounce_rate: bounce_rate(),
visit_duration: visit_duration()
}
)
@ -362,14 +372,14 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in base_session_query(site, query),
group_by: s.utm_campaign,
order_by: [desc: fragment("count"), asc: fragment("min(start)")],
order_by: [desc: uniq(s.user_id), asc: min(s.start)],
limit: ^limit,
offset: ^offset,
select: %{
name: fragment("if(empty(?), ?, ?) as name", s.utm_campaign, @no_ref, s.utm_campaign),
count: fragment("uniq(user_id) as count"),
bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"),
visit_duration: fragment("round(avg(duration * sign))")
name: coalesce_string(s.utm_campaign, @no_ref),
count: uniq(s.user_id),
bounce_rate: bounce_rate(),
visit_duration: visit_duration()
}
)
@ -392,14 +402,14 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in base_session_query(site, query),
group_by: s.utm_source,
order_by: [desc: fragment("count"), asc: fragment("min(start)")],
order_by: [desc: uniq(s.user_id), asc: min(s.start)],
limit: ^limit,
offset: ^offset,
select: %{
name: fragment("if(empty(?), ?, ?) as name", s.utm_source, @no_ref, s.utm_source),
count: fragment("uniq(user_id) as count"),
bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100)"),
visit_duration: fragment("round(avg(duration * sign))")
name: coalesce_string(s.utm_source, @no_ref),
count: uniq(s.user_id),
bounce_rate: bounce_rate(),
visit_duration: visit_duration()
}
)
@ -427,7 +437,7 @@ defmodule Plausible.Stats.Clickhouse do
join: cs in subquery(converted_sessions),
on: s.session_id == cs.session_id,
where: s.referrer_source == ^referrer,
select: fragment("uniq(user_id) as visitors")
select: uniq(s.user_id)
)
end
@ -439,7 +449,7 @@ defmodule Plausible.Stats.Clickhouse do
s in base_session_query(site, query),
group_by: s.referrer,
where: s.referrer_source == ^referrer,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: ^limit
)
|> filter_converted_sessions(site, query)
@ -449,18 +459,17 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in q,
select: %{
name: fragment("if(empty(?), ?, ?) as name", s.referrer, @no_ref, s.referrer),
count: fragment("uniq(user_id) as count"),
bounce_rate:
fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate"),
visit_duration: fragment("round(avg(duration * sign)) as visit_duration")
name: coalesce_string(s.referrer, @no_ref),
count: uniq(s.user_id),
bounce_rate: bounce_rate(),
visit_duration: visit_duration()
}
)
else
from(s in q,
select: %{
name: fragment("if(empty(?), ?, ?) as name", s.referrer, @no_ref, s.referrer),
count: fragment("uniq(user_id) as count")
name: coalesce_string(s.referrer, @no_ref),
count: uniq(s.user_id)
}
)
end
@ -495,11 +504,11 @@ defmodule Plausible.Stats.Clickhouse do
from s in base_query_w_sessions(site, query),
where: s.referrer_source == ^referrer,
group_by: s.referrer,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: 100,
select: %{
name: s.referrer,
count: fragment("uniq(user_id) as count")
count: uniq(s.user_id)
}
)
end
@ -511,14 +520,14 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in base_session_query(site, query),
group_by: s.entry_page,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: ^limit,
offset: ^offset,
select: %{
name: s.entry_page,
count: fragment("uniq(?) as count", s.user_id),
entries: fragment("uniq(?) as entries", s.session_id),
visit_duration: fragment("avg(?) as visit_duration", s.duration)
count: uniq(s.user_id),
entries: uniq(s.session_id),
visit_duration: visit_duration()
}
)
|> filter_converted_sessions(site, query)
@ -533,14 +542,14 @@ defmodule Plausible.Stats.Clickhouse do
from(
s in base_session_query(site, query),
group_by: s.exit_page,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: ^limit,
offset: ^offset,
where: s.exit_page != "",
select: %{
name: s.exit_page,
count: fragment("uniq(?) as count", s.user_id),
exits: fragment("uniq(?) as exits", s.session_id)
count: uniq(s.user_id),
exits: uniq(s.session_id)
}
)
|> filter_converted_sessions(site, query)
@ -557,7 +566,7 @@ defmodule Plausible.Stats.Clickhouse do
where: e.name == "pageview",
select: {
e.pathname,
fragment("count(*)")
total()
}
)
@ -584,12 +593,12 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from s in q,
group_by: s.exit_page,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(s.user_id)],
limit: ^limit,
offset: ^offset,
select: %{
name: fragment("? as name", s.exit_page),
count: fragment("uniq(?) as count", s.user_id)
name: s.exit_page,
count: uniq(s.user_id)
}
)
end
@ -601,13 +610,13 @@ defmodule Plausible.Stats.Clickhouse do
from(
e in base_query_w_sessions(site, query),
group_by: e.pathname,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
limit: ^limit,
offset: ^offset,
select: %{
name: fragment("? as name", e.pathname),
count: fragment("uniq(?) as count", e.user_id),
pageviews: fragment("count(*) as pageviews")
name: e.pathname,
count: uniq(e.user_id),
pageviews: total()
}
)
@ -627,12 +636,12 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from s in q,
group_by: s.entry_page,
order_by: [desc: fragment("total")],
order_by: [desc: total()],
limit: 100,
select: %{
entry_page: s.entry_page,
total: fragment("count(*) as total"),
bounce_rate: fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")
total: total(),
bounce_rate: bounce_rate()
}
)
|> Enum.map(fn row -> {row[:entry_page], row[:bounce_rate]} end)
@ -652,10 +661,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.screen_size,
where: e.screen_size != "",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.screen_size,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> add_percentages
@ -666,10 +675,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.country_code,
where: e.country_code != "\0\0",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.country_code,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> Enum.map(fn stat ->
@ -687,10 +696,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.browser,
where: e.browser != "",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.browser,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> add_percentages
@ -702,10 +711,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.browser_version,
where: e.browser_version != "",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.browser_version,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> add_percentages
@ -717,10 +726,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.operating_system,
where: e.operating_system != "",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.operating_system,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> add_percentages
@ -732,10 +741,10 @@ defmodule Plausible.Stats.Clickhouse do
from e in base_query_w_sessions(site, query),
group_by: e.operating_system_version,
where: e.operating_system_version != "",
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: e.operating_system_version,
count: fragment("uniq(user_id) as count")
count: uniq(e.user_id)
}
)
|> add_percentages
@ -744,8 +753,8 @@ defmodule Plausible.Stats.Clickhouse do
def current_visitors(site, query) do
Plausible.ClickhouseRepo.one(
from s in base_query(site, query),
select: fragment("uniq(user_id)")
from e in base_query(site, query),
select: uniq(e.user_id)
)
end
@ -801,22 +810,22 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from e in base_query_w_sessions(site, query),
where: fragment("not has(meta.key, ?)", ^key),
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: "(none)",
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
else
ClickhouseRepo.all(
from [e, meta: meta] in base_query_w_sessions(site, query),
group_by: meta.value,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: meta.value,
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
end
@ -829,8 +838,8 @@ defmodule Plausible.Stats.Clickhouse do
where: fragment("not has(?.key, ?)", e.meta, ^key),
select: %{
name: "(none)",
count: fragment("uniq(?) as count", e.user_id),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
@ -840,11 +849,11 @@ defmodule Plausible.Stats.Clickhouse do
inner_lateral_join: meta in fragment("meta as m"),
where: meta.key == ^key,
group_by: meta.value,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: meta.value,
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
@ -881,11 +890,11 @@ defmodule Plausible.Stats.Clickhouse do
ClickhouseRepo.all(
from e in base_query_w_sessions(site, query),
group_by: e.name,
order_by: [desc: fragment("count")],
order_by: [desc: uniq(e.user_id)],
select: %{
name: ^goal,
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
end
@ -912,8 +921,8 @@ defmodule Plausible.Stats.Clickhouse do
group_by: e.name,
select: %{
name: e.name,
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
}
)
@ -946,8 +955,8 @@ defmodule Plausible.Stats.Clickhouse do
),
select: %{
index: fragment("arrayJoin(indices) as index"),
count: fragment("uniq(user_id) as count"),
total_count: fragment("count(*) as total_count")
count: uniq(e.user_id),
total_count: total()
},
group_by: fragment("index")
)
@ -969,6 +978,7 @@ defmodule Plausible.Stats.Clickhouse do
sessions_q =
from(s in "sessions",
hints: ["SAMPLE 10000000"],
where: s.domain == ^site.domain,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime,
select: %{session_id: s.session_id}
@ -1081,6 +1091,7 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(e in "events",
hints: ["SAMPLE 10000000"],
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
@ -1141,6 +1152,7 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(e in "events",
hints: ["SAMPLE 10000000"],
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)
@ -1169,6 +1181,7 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(s in "sessions",
hints: ["SAMPLE 10000000"],
where: s.domain == ^site.domain,
where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
)
@ -1283,6 +1296,7 @@ defmodule Plausible.Stats.Clickhouse do
q =
from(e in "events",
hints: ["SAMPLE 10000000"],
where: e.domain == ^site.domain,
where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
)

@ -0,0 +1,43 @@
defmodule Plausible.Stats.Fragments do
defmacro uniq(user_id) do
quote do
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id))
end
end
defmacro total() do
quote do
fragment("toUInt64(round(count(*) * any(_sample_factor)))")
end
end
defmacro sample_percent() do
quote do
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
end
end
defmacro bounce_rate() do
quote do
fragment("toUInt64(round(sum(is_bounce * sign) / sum(sign) * 100))")
end
end
defmacro visit_duration() do
quote do
fragment("toUInt64(round(avg(duration * sign)))")
end
end
defmacro coalesce_string(fieldA, fieldB) do
quote do
fragment("if(empty(?), ?, ?)", unquote(fieldA), unquote(fieldB), unquote(fieldA))
end
end
defmacro __using__(_) do
quote do
import Plausible.Stats.Fragments
end
end
end

@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.from(site.timezone, params)
plot_task = Task.async(fn -> Stats.calculate_plot(site, query) end)
top_stats = fetch_top_stats(site, query)
{top_stats, sample_percent} = fetch_top_stats(site, query)
{plot, labels, present_index} = Task.await(plot_task)
json(conn, %{
@ -17,12 +17,13 @@ defmodule PlausibleWeb.Api.StatsController do
labels: labels,
present_index: present_index,
top_stats: top_stats,
interval: query.interval
interval: query.interval,
sample_percent: sample_percent
})
end
defp fetch_top_stats(site, %Query{period: "realtime"} = query) do
[
stats = [
%{
name: "Current visitors",
count: Stats.current_visitors(site, query)
@ -36,6 +37,8 @@ defmodule PlausibleWeb.Api.StatsController do
count: Stats.total_pageviews(site, query)
}
]
{stats, 100}
end
defp fetch_top_stats(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do
@ -43,7 +46,7 @@ defmodule PlausibleWeb.Api.StatsController do
prev_query = Query.shift_back(query, site)
unique_visitors = Stats.unique_visitors(site, %{query | filters: total_filter})
prev_unique_visitors = Stats.unique_visitors(site, %{prev_query | filters: total_filter})
converted_visitors = Stats.unique_visitors(site, query)
{converted_visitors, sample_percent} = Stats.unique_visitors_with_sample_percent(site, query)
prev_converted_visitors = Stats.unique_visitors(site, prev_query)
completions = Stats.total_events(site, query)
prev_completions = Stats.total_events(site, prev_query)
@ -51,7 +54,7 @@ defmodule PlausibleWeb.Api.StatsController do
conversion_rate = calculate_cr(unique_visitors, converted_visitors)
prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors)
[
stats = [
%{
name: "Unique visitors",
count: unique_visitors,
@ -73,11 +76,16 @@ defmodule PlausibleWeb.Api.StatsController do
change: percent_change(prev_conversion_rate, conversion_rate)
}
]
{stats, sample_percent}
end
defp fetch_top_stats(site, query) do
prev_query = Query.shift_back(query, site)
{pageviews, visitors} = Stats.pageviews_and_visitors(site, query)
{pageviews, visitors, sample_percent} =
Stats.pageviews_and_visitors_with_sample_percent(site, query)
{prev_pageviews, prev_visitors} = Stats.pageviews_and_visitors(site, prev_query)
bounce_rate = Stats.bounce_rate(site, query)
prev_bounce_rate = Stats.bounce_rate(site, prev_query)
@ -95,21 +103,24 @@ defmodule PlausibleWeb.Api.StatsController do
}
end
[
%{
name: "Unique visitors",
count: visitors,
change: percent_change(prev_visitors, visitors)
},
%{
name: "Total pageviews",
count: pageviews,
change: percent_change(prev_pageviews, pageviews)
},
%{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate},
visit_duration
]
|> Enum.filter(& &1)
stats =
[
%{
name: "Unique visitors",
count: visitors,
change: percent_change(prev_visitors, visitors)
},
%{
name: "Total pageviews",
count: pageviews,
change: percent_change(prev_pageviews, pageviews)
},
%{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate},
visit_duration
]
|> Enum.filter(& &1)
{stats, sample_percent}
end
defp percent_change(old_count, new_count) do

@ -11,7 +11,7 @@
"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
"cachex": {:hex, :cachex, "3.3.0", "6f2ebb8f27491fe39121bd207c78badc499214d76c695658b19d6079beeca5c2", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "d90e5ee1dde14cef33f6b187af4335b88748b72b30c038969176cd4e6ccc31a1"},
"certifi": {:hex, :certifi, "2.5.3", "70bdd7e7188c804f3a30ee0e7c99655bc35d8ac41c23e12325f36ab449b70651", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "ed516acb3929b101208a9d700062d520f3953da3b6b918d866106ffa980e1c10"},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "918e84e55072948170b5c8a2207ba394aaeb8e5b", []},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "b30ccc93a4101a25ff0bba92113e18d8a9a8b28e", []},
"clickhousex": {:git, "https://github.com/plausible/clickhousex", "0832dd4b1af1f0eba1d1018c231bf0d8d281f031", []},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.3.2", "5c2f893d05c56ae3f5e24c1b983c2d5dfb88c6d979c9287a76a7feb1e1d8d646", [:mix], [], "hexpm", "d0993402844c49539aeadb3fe46a3c9bd190f1ecf86b6f9ebd71957534c95f04"},

@ -0,0 +1,8 @@
defmodule Plausible.ClickhouseRepo.Migrations.AddSampleBy do
use Ecto.Migration
def change do
execute "ALTER TABLE events MODIFY SAMPLE BY user_id"
execute "ALTER TABLE sessions MODIFY SAMPLE BY user_id"
end
end

@ -97,14 +97,14 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")
res = json_response(conn, 200)
assert %{"name" => "Bounce rate", "percentage" => 33.0, "change" => nil} in res["top_stats"]
assert %{"name" => "Bounce rate", "percentage" => 33, "change" => nil} in res["top_stats"]
end
test "calculates average visit duration", %{conn: conn, site: site} do
conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01")
res = json_response(conn, 200)
assert %{"name" => "Visit duration", "count" => 67.0, "change" => 100} in res["top_stats"]
assert %{"name" => "Visit duration", "count" => 67, "change" => 100} in res["top_stats"]
end
end

@ -72,7 +72,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"count" => 3,
"entries" => 3,
"name" => "/",
"visit_duration" => 66.66666666666667
"visit_duration" => 67
}
]
end
@ -86,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
assert json_response(conn, 200) == [
%{
"visit_duration" => 66.66666666666667,
"visit_duration" => 67,
"count" => 3,
"entries" => 3,
"name" => "/"

Loading…
Cancel
Save