Merge branch 'master' into update_ua_inspector

pull/3957/head
hq1 3 years ago committed by GitHub
commit 0e75c2cc97
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -15,14 +15,25 @@ env:
jobs:
build:
name: Build and test
name: "Build and test (${{ matrix.mix_env }}, ${{ matrix.postgres_image }}${{ matrix.test_experimental_reduced_joins == '1' && ', experimental_reduced_joins' || '' }})"
runs-on: ubuntu-latest
strategy:
matrix:
mix_env: ['test', 'small_test']
postgres_image: ['postgres:15', 'postgres:16']
postgres_image: ['postgres:16']
test_experimental_reduced_joins: ['0']
include:
- mix_env: 'test'
postgres_image: 'postgres:15'
test_experimental_reduced_joins: '0'
- mix_env: 'test'
postgres_image: 'postgres:16'
test_experimental_reduced_joins: '1'
env:
MIX_ENV: ${{ matrix.mix_env }}
TEST_EXPERIMENTAL_REDUCED_JOINS: ${{ matrix.test_experimental_reduced_joins }}
services:
postgres:
image: ${{ matrix.postgres_image }}
@ -132,4 +143,4 @@ jobs:
- run: mix format --check-formatted
- run: mix deps.unlock --check-unused
- run: mix credo diff --from-git-merge-base origin/master
- run: mix dialyzer
- run: mix dialyzer

@ -22,6 +22,7 @@ if (container) {
conversionsOptedOut: container.dataset.conversionsOptedOut === 'true',
funnelsOptedOut: container.dataset.funnelsOptedOut === 'true',
propsOptedOut: container.dataset.propsOptedOut === 'true',
revenueGoals: JSON.parse(container.dataset.revenueGoals),
funnels: JSON.parse(container.dataset.funnels),
statsBegin: container.dataset.statsBegin,
nativeStatsBegin: container.dataset.nativeStatsBegin,

@ -59,6 +59,7 @@ export function parseQuery(querystring, site) {
'region': q.get('region'),
'city': q.get('city'),
'page': q.get('page'),
'hostname': q.get('hostname'),
'entry_page': q.get('entry_page'),
'exit_page': q.get('exit_page')
}

@ -1,19 +1,41 @@
import numberFormatter, {durationFormatter} from '../../util/number-formatter'
import { parsePrefix } from '../../util/filters'
export const METRIC_MAPPING = {
'Unique visitors (last 30 min)': 'visitors',
'Pageviews (last 30 min)': 'pageviews',
'Unique visitors': 'visitors',
'Visit duration': 'visit_duration',
'Total pageviews': 'pageviews',
'Views per visit': 'views_per_visit',
'Total visits': 'visits',
'Bounce rate': 'bounce_rate',
'Unique conversions': 'conversions',
'Total conversions': 'events',
'Conversion rate': 'conversion_rate',
'Average revenue': 'average_revenue',
'Total revenue': 'total_revenue',
export function getGraphableMetrics(query, site) {
const isRealtime = query.period === 'realtime'
const goalFilter = query.filters.goal
const pageFilter = query.filters.page
if (isRealtime && goalFilter) {
return ["visitors"]
} else if (isRealtime) {
return ["visitors", "pageviews"]
} else if (goalFilter && canGraphRevenueMetrics(goalFilter, site)) {
return ["visitors", "events", "average_revenue", "total_revenue", "conversion_rate"]
} else if (goalFilter) {
return ["visitors", "events", "conversion_rate"]
} else if (pageFilter) {
return ["visitors", "visits", "pageviews", "bounce_rate", "time_on_page"]
} else {
return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]
}
}
// Revenue metrics can only be graphed if:
// * The query is filtered by at least one revenue goal
// * All revenue goals in filter have the same currency
function canGraphRevenueMetrics(goalFilter, site) {
const goalsInFilter = parsePrefix(goalFilter).values
const revenueGoalsInFilter = site.revenueGoals.filter((rg) => {
return goalsInFilter.includes(rg.event_name)
})
const singleCurrency = revenueGoalsInFilter.every((rg) => {
return rg.currency === revenueGoalsInFilter[0].currency
})
return revenueGoalsInFilter.length > 0 && singleCurrency
}
export const METRIC_LABELS = {
@ -44,14 +66,6 @@ export const METRIC_FORMATTER = {
'average_revenue': numberFormatter,
}
export const LoadingState = {
loading: 'loading',
refreshing: 'refreshing',
loaded: 'loaded',
isLoadingOrRefreshing: function (state) { return [this.loading, this.refreshing].includes(state) },
isLoadedOrRefreshing: function (state) { return [this.loaded, this.refreshing].includes(state) }
}
const buildComparisonDataset = function(comparisonPlot) {
if (!comparisonPlot) return []

@ -6,7 +6,7 @@ import * as storage from '../../util/storage'
import { isKeyPressed } from '../../keybinding.js'
import { monthsBetweenDates } from '../../util/date.js'
export const INTERVAL_LABELS = {
const INTERVAL_LABELS = {
'minute': 'Minutes',
'hour': 'Hours',
'date': 'Days',
@ -14,11 +14,19 @@ export const INTERVAL_LABELS = {
'month': 'Months'
}
export const getStoredInterval = function(period, domain) {
function validIntervals(site, query) {
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
return ["week", "month"]
} else {
return site.validIntervalsByPeriod[query.period]
}
}
function getStoredInterval(period, domain) {
return storage.getItem(`interval__${period}__${domain}`)
}
export const storeInterval = function(period, domain, interval) {
function storeInterval(period, domain, interval) {
storage.setItem(`interval__${period}__${domain}`, interval)
}
@ -33,37 +41,47 @@ function subscribeKeybinding(element) {
}, [handleKeyPress])
}
function DropdownItem({ option, currentInterval, updateInterval }) {
return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
export const getCurrentInterval = function(site, query) {
const options = validIntervals(site, query)
const storedInterval = getStoredInterval(query.period, site.domain)
const defaultInterval = [...options].pop()
if (storedInterval && options.includes(storedInterval)) {
return storedInterval
} else {
return defaultInterval
}
}
export function IntervalPicker({ graphData, query, site, updateInterval }) {
export function IntervalPicker({ query, site, onIntervalUpdate }) {
if (query.period == 'realtime') return null
const menuElement = React.useRef(null)
subscribeKeybinding(menuElement)
const options = validIntervals(site, query)
const currentInterval = getCurrentInterval(site, query)
let currentInterval = graphData?.interval
subscribeKeybinding(menuElement)
let options = site.validIntervalsByPeriod[query.period]
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
options = ["week", "month"]
function updateInterval(interval) {
storeInterval(query.period, site.domain, interval)
onIntervalUpdate(interval)
}
if (!options.includes(currentInterval)) {
currentInterval = [...options].pop()
function renderDropdownItem(option) {
return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
}
return (
@ -85,7 +103,7 @@ export function IntervalPicker({ graphData, query, site, updateInterval }) {
leaveFrom="opacity-100 scale-100"
leaveTo="opacity-0 scale-95">
<Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static>
{options.map((option) => DropdownItem({ option, currentInterval, updateInterval }))}
{options.map(renderDropdownItem)}
</Menu.Items>
</Transition>
</>

@ -0,0 +1,247 @@
import React from 'react';
import { withRouter } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
import dateFormatter from './date-formatter';
import FadeIn from '../../fade-in';
import classNames from 'classnames';
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
}
getGraphMetric() {
let metric = this.props.graphData.metric
if (metric == 'visitors' && this.props.query.filters.goal) {
return 'conversions'
} else {
return metric
}
}
regenerateChart() {
const { graphData, query } = this.props
const metric = this.getGraphMetric()
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.maybeHopToHoveredPeriod.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
window.addEventListener('mousemove', this.repositionTooltip);
}
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
}
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
maybeHopToHoveredPeriod(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
render() {
const { graphData } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
)
}
}
export default withRouter(LineGraph)

@ -0,0 +1,17 @@
import React from "react"
export default function SamplingNotice({topStatData}) {
const samplePercent = topStatData?.samplePercent
if (samplePercent && samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
} else {
return null
}
}

@ -0,0 +1,53 @@
import React, { useState } from "react"
import * as api from '../../api'
import { getCurrentInterval } from "./interval-picker"
export default function StatsExport({site, query}) {
const [exporting, setExporting] = useState(false)
function startExport() {
setExporting(true)
document.cookie = "exporting="
pollExportReady()
}
function pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(pollExportReady, 1000)
} else {
setExporting(false)
}
}
function renderLoading() {
return (
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
)
}
function renderExportLink() {
const interval = getCurrentInterval(site, query)
const queryParams = api.serializeQuery(query, [{ interval }])
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
return (
<a href={endpoint} download onClick={startExport}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
<polyline points="7 10 12 15 17 10"></polyline>
<line x1="12" y1="15" x2="12" y2="3"></line>
</svg>
</a>
)
}
return (
<div className="w-4 h-4 mx-2">
{exporting && renderLoading()}
{!exporting && renderExportLink()}
</div>
)
}

@ -3,8 +3,9 @@ import { Tooltip } from '../../util/tooltip'
import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load'
import classNames from "classnames";
import numberFormatter, { durationFormatter } from '../../util/number-formatter'
import { METRIC_MAPPING } from './graph-util'
import * as storage from '../../util/storage'
import { formatDateRange } from '../../util/date.js'
import { getGraphableMetrics } from "./graph-util.js";
function Maybe({condition, children}) {
if (condition) {
@ -14,99 +15,103 @@ function Maybe({condition, children}) {
}
}
export default class TopStats extends React.Component {
renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg,
"pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
"pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
} else if (comparison < 0) {
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
} else if (comparison === 0) {
return <span className={noChangeClassName}>&#12336; 0%</span>
} else {
return null
}
function renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg,
"pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
"pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
} else if (comparison < 0) {
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
} else if (comparison === 0) {
return <span className={noChangeClassName}>&#12336; 0%</span>
} else {
return null
}
}
topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short
} else {
return numberFormatter(value)
}
function topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short
} else {
return numberFormatter(value)
}
}
topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long
} else {
return (value || 0).toLocaleString()
}
function topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long
} else {
return (value || 0).toLocaleString()
}
}
export default function TopStats(props) {
const {site, query, data, onMetricUpdate, tooltipBoundary, lastLoadTimestamp} = props
topStatTooltip(stat, query) {
function tooltip(stat) {
let statName = stat.name.toLowerCase()
statName = stat.value === 1 ? statName.slice(0, -1) : statName
return (
<div>
{query.comparison && <div className="whitespace-nowrap">
{this.topStatNumberLong(stat.name, stat.value)} vs. {this.topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{this.renderPercentageComparison(stat.name, stat.change, true)}</span>
{topStatNumberLong(stat.name, stat.value)} vs. {topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{renderPercentageComparison(stat.name, stat.change, true)}</span>
</div>}
{!query.comparison && <div className="whitespace-nowrap">
{this.topStatNumberLong(stat.name, stat.value)} {statName}
{topStatNumberLong(stat.name, stat.value)} {statName}
</div>}
{stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={this.props.lastLoadTimestamp}/>s ago</p>}
{stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp}/>s ago</p>}
</div>
)
}
canMetricBeGraphed(stat) {
const isTotalUniqueVisitors = this.props.query.filters.goal && stat.name === 'Unique visitors'
const isKnownMetric = Object.keys(METRIC_MAPPING).includes(stat.name)
return isKnownMetric && !isTotalUniqueVisitors
function canMetricBeGraphed(stat) {
const graphableMetrics = getGraphableMetrics(query, site)
return stat.graph_metric && graphableMetrics.includes(stat.graph_metric)
}
maybeUpdateMetric(stat) {
if (this.canMetricBeGraphed(stat)) {
this.props.updateMetric(METRIC_MAPPING[stat.name])
function maybeUpdateMetric(stat) {
if (canMetricBeGraphed(stat)) {
storage.setItem(`metric__${site.domain}`, stat.graph_metric)
onMetricUpdate(stat.graph_metric)
}
}
blinkingDot() {
function blinkingDot() {
return (
<div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div>
)
}
renderStatName(stat) {
const { metric } = this.props
const isSelected = metric === METRIC_MAPPING[stat.name]
function getStoredMetric() {
return storage.getItem(`metric__${site.domain}`)
}
function renderStatName(stat) {
const isSelected = stat.graph_metric === getStoredMetric()
const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g)
@ -123,48 +128,45 @@ export default class TopStats extends React.Component {
)
}
render() {
const { topStatData, query, site } = this.props
const stats = topStatData && topStatData.top_stats.map((stat, index) => {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': this.canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
return (
<Tooltip key={stat.name} info={this.topStatTooltip(stat, query)} className={className} onClick={() => { this.maybeUpdateMetric(stat) }} boundary={this.props.tooltipBoundary}>
{this.renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={METRIC_MAPPING[stat.name]}>{this.topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ this.renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, topStatData.from, topStatData.to) }</p>
</Maybe>
</div>
function renderStat(stat, index) {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
return (
<Tooltip key={stat.name} info={tooltip(stat, query)} className={className} onClick={() => { maybeUpdateMetric(stat) }} boundary={tooltipBoundary}>
{renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={stat.graph_metric}>{topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ this.topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, topStatData.comparing_from, topStatData.comparing_to) }</p>
</div>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, data.from, data.to) }</p>
</Maybe>
</div>
<Maybe condition={query.comparison}>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, data.comparing_from, data.comparing_to) }</p>
</div>
</Tooltip>
)
})
</Maybe>
</div>
</Tooltip>
)
}
if (stats && query && query.period === 'realtime') {
stats.push(this.blinkingDot())
}
const stats = data && data.top_stats.map(renderStat)
return stats || null;
if (stats && query.period === 'realtime') {
stats.push(blinkingDot())
}
return stats || null;
}

@ -1,534 +1,153 @@
import React from 'react';
import { withRouter, Link } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import React, { useState, useEffect, useRef, useCallback } from 'react';
import * as api from '../../api'
import * as storage from '../../util/storage'
import LazyLoader from '../../components/lazy-loader'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_MAPPING, METRIC_LABELS, METRIC_FORMATTER, LoadingState } from './graph-util'
import dateFormatter from './date-formatter';
import { getGraphableMetrics } from './graph-util'
import TopStats from './top-stats';
import { IntervalPicker, getStoredInterval, storeInterval } from './interval-picker';
import { IntervalPicker, getCurrentInterval } from './interval-picker'
import StatsExport from './stats-export'
import WithImportedSwitch from './with-imported-switch';
import SamplingNotice from './sampling-notice';
import FadeIn from '../../fade-in';
import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input'
import LineGraphWithRouter from './line-graph'
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.boundary = React.createRef()
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
this.state = {
exported: false
};
function fetchTopStats(site, query) {
const q = { ...query }
if (!isComparisonEnabled(q.comparison)) {
q.comparison = 'previous_period'
}
regenerateChart() {
const { graphData, metric, query } = this.props
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.onClick.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
return api.get(url.apiPath(site, '/top-stats'), q)
}
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
function fetchMainGraph(site, query, metric, interval) {
const params = {metric, interval}
return api.get(url.apiPath(site, '/main-graph'), query, params)
}
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
export default function VisitorGraph(props) {
const {site, query, lastLoadTimestamp} = props
const isRealtime = query.period === 'realtime'
const isDarkTheme = document.querySelector('html').classList.contains('dark') || false
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
const topStatsBoundary = useRef(null)
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
window.addEventListener('mousemove', this.repositionTooltip);
}
const [topStatData, setTopStatData] = useState(null)
const [topStatsLoading, setTopStatsLoading] = useState(true)
const [graphData, setGraphData] = useState(null)
const [graphLoading, setGraphLoading] = useState(true)
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
// This state is explicitly meant for the situation where either graph interval
// or graph metric is changed. That results in behaviour where Top Stats stay
// intact, but the graph container alone will display a loading spinner for as
// long as new graph data is fetched.
const [graphRefreshing, setGraphRefreshing] = useState(false)
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
const onIntervalUpdate = useCallback((newInterval) => {
setGraphData(null)
setGraphRefreshing(true)
fetchGraphData(getStoredMetric(), newInterval)
}, [query])
if (tooltip) {
tooltip.style.display = 'none';
}
}
const onMetricUpdate = useCallback((newMetric) => {
setGraphData(null)
setGraphRefreshing(true)
fetchGraphData(newMetric, getCurrentInterval(site, query))
}, [query])
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
useEffect(() => {
setTopStatData(null)
setTopStatsLoading(true)
setGraphData(null)
setGraphLoading(true)
fetchTopStatsAndGraphData()
if (tooltip) {
tooltip.style.display = 'none';
}
if (isRealtime) {
document.addEventListener('tick', fetchTopStatsAndGraphData)
}
}
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
return () => {
document.removeEventListener('tick', fetchTopStatsAndGraphData)
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
}, [query])
onClick(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
useEffect(() => {
if (topStatData) { storeTopStatsContainerHeight() }
}, [topStatData])
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
function fetchTopStatsAndGraphData() {
fetchTopStats(site, query)
.then((res) => {
setTopStatData(res)
setTopStatsLoading(false)
})
let metric = getStoredMetric()
const availableMetrics = getGraphableMetrics(query, site)
if (!availableMetrics.includes(metric)) {
metric = availableMetrics[0]
storage.setItem(`metric__${site.domain}`, metric)
}
}
pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(this.pollExportReady.bind(this), 1000);
} else {
this.setState({ exported: false })
}
}
const interval = getCurrentInterval(site, query)
downloadSpinner() {
this.setState({ exported: true });
document.cookie = "exporting=";
setTimeout(this.pollExportReady.bind(this), 1000);
fetchGraphData(metric, interval)
}
downloadLink() {
if (this.props.query.period !== 'realtime') {
if (this.state.exported) {
return (
<div className="w-4 h-4 mx-2">
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
</div>
)
} else {
const interval = this.props.graphData?.interval
const queryParams = api.serializeQuery(this.props.query, [{ interval }])
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${queryParams}`
return (
<a className="w-4 h-4 mx-2" href={endpoint} download onClick={this.downloadSpinner.bind(this)}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
</a>
)
}
}
function fetchGraphData(metric, interval) {
fetchMainGraph(site, query, metric, interval)
.then((res) => {
setGraphData(res)
setGraphLoading(false)
setGraphRefreshing(false)
})
}
samplingNotice() {
const samplePercent = this.props.topStatData && this.props.topStatData.sample_percent
if (samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
}
function getStoredMetric() {
return storage.getItem(`metric__${site.domain}`)
}
importedNotice() {
if (!this.props.topStatData?.imported_source) return
const isBeforeNativeStats = (date) => {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(this.props.site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(this.props.topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
const strike = withImported ? "" : " line-through"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
<svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
</svg>
</div>
</Link>
)
}
function storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${site.domain}`, document.getElementById('top-stats-container').clientHeight)
}
// This function is used for maintaining the main-graph/top-stats container height in the
// loading process. The container height depends on how many top stat metrics are returned
// from the API, but in the loading state, we don't know that yet. We can use localStorage
// to keep track of the Top Stats container height.
getTopStatsHeight() {
if (this.props.topStatData) {
function getTopStatsHeight() {
if (topStatData) {
return 'auto'
} else {
return `${storage.getItem(`topStatsHeight__${this.props.site.domain}`) || 89}px`
return `${storage.getItem(`topStatsHeight__${site.domain}`) || 89}px`
}
}
render() {
const { mainGraphRefreshing, updateMetric, updateInterval, metric, topStatData, query, site, graphData, lastLoadTimestamp } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<div>
<div id="top-stats-container" className="flex flex-wrap" ref={this.boundary} style={{ height: this.getTopStatsHeight() }}>
<TopStats site={site} query={query} metric={metric} updateMetric={updateMetric} topStatData={topStatData} tooltipBoundary={this.boundary.current} lastLoadTimestamp={lastLoadTimestamp} />
return (
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{(topStatsLoading || graphLoading) && renderLoader()}
<FadeIn show={!(topStatsLoading || graphLoading)}>
<div id="top-stats-container" className="flex flex-wrap" ref={topStatsBoundary} style={{ height: getTopStatsHeight() }}>
<TopStats site={site} query={query} data={topStatData} onMetricUpdate={onMetricUpdate} tooltipBoundary={topStatsBoundary.current} lastLoadTimestamp={lastLoadTimestamp} />
</div>
<div className="relative px-2">
{mainGraphRefreshing && renderLoader()}
{graphRefreshing && renderLoader()}
<div className="absolute right-4 -top-8 py-1 flex items-center">
{this.downloadLink()}
{this.samplingNotice()}
{this.importedNotice()}
<IntervalPicker site={site} query={query} graphData={graphData} metric={metric} updateInterval={updateInterval} />
{!isRealtime && <StatsExport site={site} query={query} />}
<SamplingNotice samplePercent={topStatData}/>
<WithImportedSwitch site={site} topStatData={topStatData} />
<IntervalPicker site={site} query={query} onIntervalUpdate={onIntervalUpdate} />
</div>
<FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
<LineGraphWithRouter graphData={graphData} darkTheme={isDarkTheme} query={query} />
</div>
</div>
)
}
}
const LineGraphWithRouter = withRouter(LineGraph)
export default class VisitorGraph extends React.Component {
constructor(props) {
super(props)
this.state = {
topStatsLoadingState: LoadingState.loading,
mainGraphLoadingState: LoadingState.loading,
metric: storage.getItem(`metric__${this.props.site.domain}`) || 'visitors'
}
this.onVisible = this.onVisible.bind(this)
this.updateMetric = this.updateMetric.bind(this)
this.fetchTopStatData = this.fetchTopStatData.bind(this)
this.fetchGraphData = this.fetchGraphData.bind(this)
this.updateInterval = this.updateInterval.bind(this)
}
isIntervalValid(interval) {
const { query, site } = this.props
const validIntervals = site.validIntervalsByPeriod[query.period] || []
return validIntervals.includes(interval)
}
getIntervalFromStorage() {
const { query, site } = this.props
let interval = getStoredInterval(query.period, site.domain)
if (interval !== "week" && interval !== "month" && query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
interval = "month"
}
if (this.isIntervalValid(interval)) {
return interval
} else {
return null
}
}
updateInterval(interval) {
if (this.isIntervalValid(interval)) {
storeInterval(this.props.query.period, this.props.site.domain, interval)
this.setState({ mainGraphLoadingState: LoadingState.refreshing, graphData: null }, this.fetchGraphData)
}
}
onVisible() {
this.setState({ mainGraphLoadingState: LoadingState.loading }, this.fetchGraphData)
this.fetchTopStatData()
if (this.props.query.period === 'realtime') {
document.addEventListener('tick', this.fetchGraphData)
document.addEventListener('tick', this.fetchTopStatData)
}
}
componentDidUpdate(prevProps, prevState) {
const { metric } = this.state;
const { query } = this.props
if (query !== prevProps.query) {
this.setState({ mainGraphLoadingState: LoadingState.loading, topStatsLoadingState: LoadingState.loading, graphData: null, topStatData: null }, this.fetchGraphData)
this.fetchTopStatData()
}
if (metric !== prevState.metric) {
this.setState({ mainGraphLoadingState: LoadingState.refreshing }, this.fetchGraphData)
}
}
resetMetric() {
const { topStatData } = this.state
const { query, site } = this.props
const savedMetric = storage.getItem(`metric__${site.domain}`)
const selectableMetrics = topStatData && topStatData.top_stats.map(({ name }) => METRIC_MAPPING[name]).filter(name => name)
const canSelectSavedMetric = selectableMetrics && selectableMetrics.includes(savedMetric)
if (query.filters.goal && !['conversion_rate', 'events'].includes(savedMetric)) {
this.setState({ metric: 'conversions' })
} else if (canSelectSavedMetric) {
this.setState({ metric: savedMetric })
} else {
this.setState({ metric: 'visitors' })
}
}
componentWillUnmount() {
document.removeEventListener('tick', this.fetchGraphData)
document.removeEventListener('tick', this.fetchTopStatData)
}
storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${this.props.site.domain}`, document.getElementById('top-stats-container').clientHeight)
}
updateMetric(clickedMetric) {
if (this.state.metric == clickedMetric) return
storage.setItem(`metric__${this.props.site.domain}`, clickedMetric)
this.setState({ metric: clickedMetric, graphData: null })
}
fetchGraphData() {
const url = `/api/stats/${encodeURIComponent(this.props.site.domain)}/main-graph`
let params = { metric: this.state.metric }
const interval = this.getIntervalFromStorage()
if (interval) { params.interval = interval }
api.get(url, this.props.query, params)
.then((res) => {
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: res })
return res
})
.catch((err) => {
console.log(err)
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: false })
})
}
fetchTopStatData() {
const query = { ...this.props.query }
if (!isComparisonEnabled(query.comparison)) query.comparison = 'previous_period'
api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/top-stats`, query)
.then((res) => {
this.setState({ topStatsLoadingState: LoadingState.loaded, topStatData: res }, () => {
this.storeTopStatsContainerHeight()
this.resetMetric()
})
return res
})
}
renderInner() {
const { query, site } = this.props;
const { graphData, metric, topStatData, topStatsLoadingState, mainGraphLoadingState } = this.state;
const theme = document.querySelector('html').classList.contains('dark') || false
const mainGraphRefreshing = (mainGraphLoadingState === LoadingState.refreshing)
const topStatAndGraphLoaded = !!(topStatData && graphData)
const shouldShow =
topStatsLoadingState === LoadingState.loaded &&
LoadingState.isLoadedOrRefreshing(mainGraphLoadingState) &&
(topStatData && mainGraphRefreshing || topStatAndGraphLoaded)
return (
<FadeIn show={shouldShow}>
<LineGraphWithRouter mainGraphRefreshing={mainGraphRefreshing} graphData={graphData} topStatData={topStatData} site={site} query={query} darkTheme={theme} metric={metric} updateMetric={this.updateMetric} updateInterval={this.updateInterval} lastLoadTimestamp={this.props.lastLoadTimestamp} />
</FadeIn>
)
}
render() {
const { mainGraphLoadingState, topStatsLoadingState } = this.state
const showLoader =
[mainGraphLoadingState, topStatsLoadingState].includes(LoadingState.loading) &&
mainGraphLoadingState !== LoadingState.refreshing
return (
<LazyLoader onVisible={this.onVisible}>
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{showLoader && renderLoader()}
{this.renderInner()}
</div>
</LazyLoader>
)
}
</div>
)
}
function renderLoader() {

@ -0,0 +1,40 @@
import React from "react"
import { parseNaiveDate, isBefore } from '../../util/date'
import { Link } from 'react-router-dom'
import * as url from '../../util/url'
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
export default function WithImportedSwitch({site, topStatData}) {
if (!topStatData?.imports_exist) {
return null
}
function isBeforeNativeStats(date) {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const withImported = topStatData.with_imported;
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
<BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)
} else {
return null
}
}

@ -1,3 +1,5 @@
const flags = JSON.parse(document.getElementById('stats-react-container').dataset.flags)
export const FILTER_GROUPS = {
'page': ['page', 'entry_page', 'exit_page'],
'source': ['source', 'referrer'],
@ -7,7 +9,9 @@ export const FILTER_GROUPS = {
'os': ['os', 'os_version'],
'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'],
'goal': ['goal'],
'props': ['prop_key', 'prop_value']
'props': ['prop_key', 'prop_value'],
...(flags.hostname_filter ? { 'hostname': ['hostname'] } : {})
}
export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location']))
@ -37,7 +41,7 @@ export function isFreeChoiceFilter(filterName) {
let NON_ESCAPED_PIPE_REGEX;
try {
NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g")
} catch(_e) {
} catch (_e) {
NON_ESCAPED_PIPE_REGEX = '|'
}
@ -64,19 +68,19 @@ export function parsePrefix(rawValue) {
.filter((clause) => !!clause)
.map((val) => val.replaceAll(ESCAPED_PIPE, '|'))
return {type, values}
return { type, values }
}
export function parseQueryPropsFilter(query) {
return Object.entries(query.filters['props']).map(([key, propVal]) => {
const {type, values} = parsePrefix(propVal)
const clauses = values.map(val => { return {value: val, label: val}})
const { type, values } = parsePrefix(propVal)
const clauses = values.map(val => { return { value: val, label: val } })
return { propKey: { label: key, value: key }, type, clauses }
})
}
export function parseQueryFilter(query, filter) {
const {type, values} = parsePrefix(query.filters[filter] || '')
const { type, values } = parsePrefix(query.filters[filter] || '')
let labels = values
@ -95,9 +99,9 @@ export function parseQueryFilter(query, filter) {
labels = rawLabel.split('|').filter(label => !!label)
}
const clauses = values.map((value, index) => { return {value, label: labels[index]}})
const clauses = values.map((value, index) => { return { value, label: labels[index] } })
return {type, clauses}
return { type, clauses }
}
export function isFilteringOnFixedValue(query, filter) {
@ -152,6 +156,7 @@ export const formattedFilters = {
'region': 'Region',
'city': 'City',
'page': 'Page',
'hostname': 'Hostname',
'entry_page': 'Entry Page',
'exit_page': 'Exit Page'
'exit_page': 'Exit Page',
}

@ -0,0 +1,38 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240302"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -10,7 +10,7 @@
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
"timeZone": "Europe/Amsterdam"
},
"metricHeaders": [
{
@ -34,77 +34,749 @@
"type": "TYPE_SECONDS"
}
],
"rowCount": 3,
"rowCount": 31,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
"value": "20240131"
}
],
"metricValues": [
{
"value": "1"
"value": "141"
},
{
"value": "1"
"value": "195"
},
{
"value": "1"
"value": "56"
},
{
"value": "1"
"value": "146"
},
{
"value": "0"
"value": "4815"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
"value": "20240130"
}
],
"metricValues": [
{
"value": "2"
"value": "115"
},
{
"value": "2"
"value": "171"
},
{
"value": "2"
"value": "42"
},
{
"value": "2"
"value": "115"
},
{
"value": "0"
"value": "3481"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
"value": "20240129"
}
],
"metricValues": [
{
"value": "3"
"value": "105"
},
{
"value": "13"
"value": "153"
},
{
"value": "0"
"value": "54"
},
{
"value": "4"
"value": "113"
},
{
"value": "29"
"value": "2895"
}
]
},
{
"dimensionValues": [
{
"value": "20240128"
}
],
"metricValues": [
{
"value": "184"
},
{
"value": "259"
},
{
"value": "74"
},
{
"value": "190"
},
{
"value": "6861"
}
]
},
{
"dimensionValues": [
{
"value": "20240127"
}
],
"metricValues": [
{
"value": "155"
},
{
"value": "202"
},
{
"value": "67"
},
{
"value": "158"
},
{
"value": "5259"
}
]
},
{
"dimensionValues": [
{
"value": "20240126"
}
],
"metricValues": [
{
"value": "132"
},
{
"value": "189"
},
{
"value": "57"
},
{
"value": "142"
},
{
"value": "5545"
}
]
},
{
"dimensionValues": [
{
"value": "20240125"
}
],
"metricValues": [
{
"value": "151"
},
{
"value": "229"
},
{
"value": "71"
},
{
"value": "155"
},
{
"value": "5365"
}
]
},
{
"dimensionValues": [
{
"value": "20240124"
}
],
"metricValues": [
{
"value": "137"
},
{
"value": "221"
},
{
"value": "64"
},
{
"value": "148"
},
{
"value": "4454"
}
]
},
{
"dimensionValues": [
{
"value": "20240123"
}
],
"metricValues": [
{
"value": "166"
},
{
"value": "267"
},
{
"value": "61"
},
{
"value": "171"
},
{
"value": "4850"
}
]
},
{
"dimensionValues": [
{
"value": "20240122"
}
],
"metricValues": [
{
"value": "161"
},
{
"value": "233"
},
{
"value": "73"
},
{
"value": "165"
},
{
"value": "4871"
}
]
},
{
"dimensionValues": [
{
"value": "20240121"
}
],
"metricValues": [
{
"value": "254"
},
{
"value": "397"
},
{
"value": "114"
},
{
"value": "267"
},
{
"value": "8845"
}
]
},
{
"dimensionValues": [
{
"value": "20240120"
}
],
"metricValues": [
{
"value": "214"
},
{
"value": "383"
},
{
"value": "100"
},
{
"value": "223"
},
{
"value": "6914"
}
]
},
{
"dimensionValues": [
{
"value": "20240119"
}
],
"metricValues": [
{
"value": "381"
},
{
"value": "5571"
},
{
"value": "100"
},
{
"value": "228"
},
{
"value": "98549"
}
]
},
{
"dimensionValues": [
{
"value": "20240118"
}
],
"metricValues": [
{
"value": "177"
},
{
"value": "278"
},
{
"value": "86"
},
{
"value": "186"
},
{
"value": "5553"
}
]
},
{
"dimensionValues": [
{
"value": "20240117"
}
],
"metricValues": [
{
"value": "204"
},
{
"value": "271"
},
{
"value": "97"
},
{
"value": "216"
},
{
"value": "6777"
}
]
},
{
"dimensionValues": [
{
"value": "20240116"
}
],
"metricValues": [
{
"value": "200"
},
{
"value": "280"
},
{
"value": "65"
},
{
"value": "203"
},
{
"value": "9381"
}
]
},
{
"dimensionValues": [
{
"value": "20240115"
}
],
"metricValues": [
{
"value": "255"
},
{
"value": "311"
},
{
"value": "86"
},
{
"value": "269"
},
{
"value": "12365"
}
]
},
{
"dimensionValues": [
{
"value": "20240114"
}
],
"metricValues": [
{
"value": "243"
},
{
"value": "302"
},
{
"value": "86"
},
{
"value": "257"
},
{
"value": "11073"
}
]
},
{
"dimensionValues": [
{
"value": "20240113"
}
],
"metricValues": [
{
"value": "231"
},
{
"value": "276"
},
{
"value": "92"
},
{
"value": "244"
},
{
"value": "10084"
}
]
},
{
"dimensionValues": [
{
"value": "20240112"
}
],
"metricValues": [
{
"value": "215"
},
{
"value": "265"
},
{
"value": "75"
},
{
"value": "225"
},
{
"value": "10195"
}
]
},
{
"dimensionValues": [
{
"value": "20240111"
}
],
"metricValues": [
{
"value": "183"
},
{
"value": "235"
},
{
"value": "64"
},
{
"value": "191"
},
{
"value": "8200"
}
]
},
{
"dimensionValues": [
{
"value": "20240110"
}
],
"metricValues": [
{
"value": "190"
},
{
"value": "244"
},
{
"value": "61"
},
{
"value": "203"
},
{
"value": "9056"
}
]
},
{
"dimensionValues": [
{
"value": "20240109"
}
],
"metricValues": [
{
"value": "201"
},
{
"value": "290"
},
{
"value": "63"
},
{
"value": "213"
},
{
"value": "11651"
}
]
},
{
"dimensionValues": [
{
"value": "20240108"
}
],
"metricValues": [
{
"value": "195"
},
{
"value": "235"
},
{
"value": "71"
},
{
"value": "209"
},
{
"value": "7946"
}
]
},
{
"dimensionValues": [
{
"value": "20240107"
}
],
"metricValues": [
{
"value": "205"
},
{
"value": "274"
},
{
"value": "80"
},
{
"value": "218"
},
{
"value": "10048"
}
]
},
{
"dimensionValues": [
{
"value": "20240106"
}
],
"metricValues": [
{
"value": "256"
},
{
"value": "303"
},
{
"value": "119"
},
{
"value": "281"
},
{
"value": "8440"
}
]
},
{
"dimensionValues": [
{
"value": "20240105"
}
],
"metricValues": [
{
"value": "226"
},
{
"value": "270"
},
{
"value": "83"
},
{
"value": "237"
},
{
"value": "9311"
}
]
},
{
"dimensionValues": [
{
"value": "20240104"
}
],
"metricValues": [
{
"value": "223"
},
{
"value": "274"
},
{
"value": "81"
},
{
"value": "236"
},
{
"value": "12051"
}
]
},
{
"dimensionValues": [
{
"value": "20240103"
}
],
"metricValues": [
{
"value": "233"
},
{
"value": "283"
},
{
"value": "79"
},
{
"value": "245"
},
{
"value": "10628"
}
]
},
{
"dimensionValues": [
{
"value": "20240102"
}
],
"metricValues": [
{
"value": "203"
},
{
"value": "269"
},
{
"value": "60"
},
{
"value": "210"
},
{
"value": "11168"
}
]
},
{
"dimensionValues": [
{
"value": "20240101"
}
],
"metricValues": [
{
"value": "191"
},
{
"value": "224"
},
{
"value": "71"
},
{
"value": "197"
},
{
"value": "8080"
}
]
}

@ -0,0 +1,38 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20200421"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -0,0 +1,38 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20170118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -24,6 +24,7 @@ defmodule Mix.Tasks.SendPageview do
page: :string,
referrer: :string,
host: :string,
hostname: :string,
event: :string,
props: :string,
revenue_currency: :string,
@ -86,6 +87,7 @@ defmodule Mix.Tasks.SendPageview do
referrer = Keyword.get(opts, :referrer, @default_referrer)
event = Keyword.get(opts, :event, @default_event)
props = Keyword.get(opts, :props, @default_props)
hostname = Keyword.get(opts, :hostname, domain)
revenue =
if Keyword.get(opts, :revenue_currency) do
@ -97,7 +99,7 @@ defmodule Mix.Tasks.SendPageview do
%{
name: event,
url: "http://#{domain}#{page}",
url: "http://#{hostname}#{page}",
domain: domain,
referrer: referrer,
props: props,

@ -65,4 +65,28 @@ defmodule Plausible.ClickhouseEventV2 do
)
|> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
end
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
def merge_session(%__MODULE__{} = event, session) do
Map.merge(event, Map.take(session, @session_properties))
end
end

@ -43,6 +43,7 @@ defmodule Plausible.ClickhouseSessionV2 do
field :is_bounce, BoolUInt8
field :entry_page, :string
field :exit_page, :string
field :exit_page_hostname, :string
field :pageviews, Ch, type: "Int32"
field :events, Ch, type: "Int32"
field :sign, Ch, type: "Int8"

@ -0,0 +1,171 @@
defmodule Plausible.DataMigration.SiteImports do
@moduledoc """
Site imports migration backfilling SiteImport entries for old imports
and alters import end dates to match actual end date of respective import stats.
"""
import Ecto.Query
alias Plausible.ClickhouseRepo
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
def run(opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
sites_with_imports =
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|> Repo.all(log: false)
sites_count = length(sites_with_imports)
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
for {site, idx} <- Enum.with_index(sites_with_imports) do
site_imports =
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|> Repo.all(log: false)
IO.puts(
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
)
site_imports =
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
IO.puts("Creating legacy site import entry for site ID #{site.id}")
# create legacy entry if there's not one yet
params =
site.imported_data
|> Imported.SiteImport.from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
legacy_site_import =
%Imported.SiteImport{}
|> Ecto.Changeset.change(params)
|> insert!(dry_run?)
[legacy_site_import | site_imports]
else
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
site_imports
end
# adjust end date for each site import
for site_import <- site_imports do
IO.puts(
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
)
import_ids =
if site_import.legacy do
[0, site_import.id]
else
[site_import.id]
end
end_date = imported_stats_end_date(site.id, import_ids)
end_date =
if !end_date do
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Setting end date to minimum."
)
Date.add(site_import.start_date, 2)
else
end_date
end
end_date =
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
end_date
else
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
)
site_import.end_date
end
site_import
|> Ecto.Changeset.change(end_date: end_date)
|> update!(dry_run?)
IO.puts(
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
)
end
IO.puts("Done processing site ID #{site.id}")
end
IO.puts("Finished")
end
# Exposed for testing purposes
@doc false
def imported_stats_end_date(site_id, import_ids) do
[first_schema | schemas] = Imported.schemas()
query =
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
end)
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
if dates != [] do
case Enum.max(dates, Date) do
# no stats for this domain yet
~D[1970-01-01] ->
nil
date ->
date
end
else
nil
end
end
defp insert!(changeset, false = _dry_run?) do
Repo.insert!(changeset)
end
defp insert!(changeset, true = _dry_run?) do
if changeset.valid? do
changeset
|> Ecto.Changeset.change(id: 0)
|> Ecto.Changeset.apply_changes()
else
raise "Invalid insert: #{inspect(changeset)}"
end
end
defp update!(changeset, false = _dry_run?) do
Repo.update!(changeset)
end
defp update!(changeset, true = _dry_run?) do
if changeset.valid? do
Ecto.Changeset.apply_changes(changeset)
else
raise "Invalid update: #{inspect(changeset)}"
end
end
defp max_date_query(schema, site_id, import_ids) do
from(q in schema,
where: q.site_id == ^site_id,
where: q.import_id in ^import_ids,
select: %{max_date: fragment("max(?)", q.date)}
)
end
end

@ -118,51 +118,27 @@ defmodule Plausible.Exports do
end
end
defmacrop pageviews(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
:pageviews
)
end
end
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
def export_visitors_q(site_id) do
visitors_sessions_q =
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: selected_as(:date),
select: %{
date: date(s.start),
bounces: bounces(s),
visits: visits(s),
visit_duration: visit_duration(s)
# NOTE: can we use just sessions_v2 table in this query? sum(pageviews) and visitors(s)?
# visitors: visitors(s)
}
visitors_events_q =
from e in sampled("events_v2"),
where: e.site_id == ^site_id,
group_by: selected_as(:date),
select: %{
date: date(e.timestamp),
visitors: visitors(e),
pageviews:
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
:pageviews
)
}
visitors_q =
"e"
|> with_cte("e", as: ^visitors_events_q)
|> with_cte("s", as: ^visitors_sessions_q)
from e in visitors_q,
full_join: s in "s",
on: e.date == s.date,
order_by: selected_as(:date),
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: selected_as(:date),
select: [
selected_as(fragment("greatest(?,?)", s.date, e.date), :date),
e.visitors,
e.pageviews,
s.bounces,
s.visits,
s.visit_duration
date(s.start),
visitors(s),
pageviews(s),
bounces(s),
visits(s),
visit_duration(s)
]
end
@ -173,6 +149,8 @@ defmodule Plausible.Exports do
group_by: [
selected_as(:date),
selected_as(:source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
@ -182,10 +160,13 @@ defmodule Plausible.Exports do
select: [
date(s.start),
selected_as(s.referrer_source, :source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
s.utm_term,
pageviews(s),
visitors(s),
visits(s),
visit_duration(s),
@ -210,6 +191,7 @@ defmodule Plausible.Exports do
hostname: e.hostname,
name: e.name,
user_id: e.user_id,
session_id: e.session_id,
_sample_factor: fragment("_sample_factor")
}
@ -220,6 +202,10 @@ defmodule Plausible.Exports do
date(e.timestamp),
selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page),
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
:visits
),
visitors(e),
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
@ -252,7 +238,8 @@ defmodule Plausible.Exports do
:entrances
),
visit_duration(s),
bounces(s)
bounces(s),
pageviews(s)
]
end
@ -266,10 +253,13 @@ defmodule Plausible.Exports do
date(s.start),
s.exit_page,
visitors(s),
visit_duration(s),
selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:exits
)
),
bounces(s),
pageviews(s)
]
end
@ -288,7 +278,8 @@ defmodule Plausible.Exports do
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
bounces(s),
pageviews(s)
]
end
@ -304,7 +295,8 @@ defmodule Plausible.Exports do
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
bounces(s),
pageviews(s)
]
end
@ -312,15 +304,17 @@ defmodule Plausible.Exports do
def export_browsers_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.browser],
group_by: [selected_as(:date), s.browser, s.browser_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.browser,
s.browser_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
bounces(s),
pageviews(s)
]
end
@ -328,15 +322,17 @@ defmodule Plausible.Exports do
def export_operating_systems_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.operating_system],
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.operating_system,
s.operating_system_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
bounces(s),
pageviews(s)
]
end

@ -80,6 +80,14 @@ defmodule Plausible.Goals do
def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"}
def list_revenue_goals(site) do
from(g in Plausible.Goal,
where: g.site_id == ^site.id and not is_nil(g.currency),
select: %{event_name: g.event_name, currency: g.currency}
)
|> Plausible.Repo.all()
end
def for_site(site, opts \\ []) do
site
|> for_site_query(opts)

@ -55,6 +55,14 @@ defmodule Plausible.Google.API do
end
end
def get_analytics_end_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view)
end
end
def fetch_verified_properties(auth) do
with {:ok, access_token} <- maybe_refresh_token(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do

@ -64,6 +64,10 @@ defmodule Plausible.Google.GA4.API do
GA4.HTTP.get_analytics_start_date(access_token, property)
end
def get_analytics_end_date(access_token, property) do
GA4.HTTP.get_analytics_end_date(access_token, property)
end
def import_analytics(date_range, property, auth, persist_fn) do
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"

@ -141,7 +141,7 @@ defmodule Plausible.Google.GA4.HTTP do
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error})
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
{:error, :unknown}
end
end
@ -162,7 +162,7 @@ defmodule Plausible.Google.GA4.HTTP do
{:error, :not_found}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error retrieving Google property #{property}",
Sentry.capture_message("Error retrieving GA4 property #{property}",
extra: %{error: error}
)
@ -171,7 +171,18 @@ defmodule Plausible.Google.GA4.HTTP do
end
@earliest_valid_date "2015-08-14"
def get_analytics_start_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :start)
end
def get_analytics_end_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :end)
end
defp get_analytics_boundary_date(access_token, property, edge) do
descending? = edge == :end
params = %{
requests: [
%{
@ -182,7 +193,7 @@ defmodule Plausible.Google.GA4.HTTP do
dimensions: [%{name: "date"}],
metrics: [%{name: "screenPageViews"}],
orderBys: [
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false}
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
],
limit: 1
}
@ -207,13 +218,15 @@ defmodule Plausible.Google.GA4.HTTP do
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error retrieving GA4 #{edge} date",
extra: %{error: error}
)
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)})
{:error, reason}
{:error, :unknown}
end
end

@ -46,9 +46,11 @@ defmodule Plausible.Google.GA4.ReportRequest do
"sessionMedium",
"sessionCampaignName",
"sessionManualAdContent",
"sessionGoogleAdsKeyword"
"sessionGoogleAdsKeyword",
"pageReferrer"
],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
@ -59,12 +61,13 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"]
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
},
%__MODULE__{
dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"userEngagementDuration",
@ -75,12 +78,19 @@ defmodule Plausible.Google.GA4.ReportRequest do
# %__MODULE__{
# dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"],
# metrics: ["totalUsers", "sessions"]
# metrics: [
# "totalUsers",
# "sessions",
# "screenPageViews",
# "userEngagementDuration",
# "bounces = sessions - engagedSessions"
# ]
# },
%__MODULE__{
dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
@ -91,6 +101,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_devices",
dimensions: ["date", "deviceCategory"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
@ -101,6 +112,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_browsers",
dimensions: ["date", "browser"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
@ -109,8 +121,9 @@ defmodule Plausible.Google.GA4.ReportRequest do
},
%__MODULE__{
dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem"],
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",

@ -64,6 +64,10 @@ defmodule Plausible.Google.UA.API do
UA.HTTP.get_analytics_start_date(access_token, view_id)
end
def get_analytics_end_date(access_token, view_id) do
UA.HTTP.get_analytics_end_date(access_token, view_id)
end
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()}
@doc """

@ -109,13 +109,29 @@ defmodule Plausible.Google.UA.HTTP do
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
{:error, :unknown}
end
end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :start)
end
def get_analytics_end_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :end)
end
defp get_analytics_boundary_date(access_token, view_id, edge) do
sort_order =
if edge == :start do
"ASCENDING"
else
"DESCENDING"
end
params = %{
reportRequests: [
%{
@ -127,7 +143,7 @@ defmodule Plausible.Google.UA.HTTP do
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}],
pageSize: 1
}
]
@ -151,13 +167,15 @@ defmodule Plausible.Google.UA.HTTP do
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)})
{:error, reason}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error retrieving UA #{edge} date",
extra: %{error: error}
)
{:error, :unknown}
end
end

@ -37,6 +37,9 @@ defmodule Plausible.Imported do
# Maximum number of complete imports to account for when querying stats
@max_complete_imports 5
@spec schemas() :: [module()]
def schemas, do: @tables
@spec tables() :: [String.t()]
def tables, do: @table_names
@ -48,13 +51,13 @@ defmodule Plausible.Imported do
def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site)
earliest_import = get_earliest_import(site) || %{}
dates = get_imports_date_range(site)
%{
site
| import_data_loaded: true,
earliest_import_start_date: Map.get(earliest_import, :start_date),
earliest_import_end_date: Map.get(earliest_import, :end_date),
earliest_import_start_date: dates.start_date,
latest_import_end_date: dates.end_date,
complete_import_ids: complete_import_ids
}
end
@ -66,10 +69,11 @@ defmodule Plausible.Imported do
defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t()) :: [SiteImport.t()]
def list_all_imports(site) do
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
def list_all_imports(site, status \\ nil) do
imports =
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|> maybe_filter_by_status(status)
|> Repo.all()
if site.imported_data && not Enum.any?(imports, & &1.legacy) do
@ -79,42 +83,62 @@ defmodule Plausible.Imported do
end
end
defp maybe_filter_by_status(query, nil), do: query
defp maybe_filter_by_status(query, status) do
where(query, [i], i.status == ^status)
end
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
def list_complete_import_ids(site) do
ids =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: i.id,
select: {i.legacy, i.id},
limit: @max_complete_imports
)
|> Repo.all()
has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)
ids = Enum.map(ids, &elem(&1, 1))
# account for legacy imports as well
if site.imported_data && site.imported_data.status == "ok" do
if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do
[0 | ids]
else
ids
end
end
@spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
def get_earliest_import(site) do
first_import =
@spec get_imports_date_range(Site.t()) :: %{
start_date: Date.t() | nil,
end_date: Date.t() | nil
}
def get_imports_date_range(site) do
dates =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
order_by: i.start_date,
limit: 1
select: %{start_date: min(i.start_date), end_date: max(i.end_date)}
)
|> Repo.one()
legacy_import =
if site.imported_data && site.imported_data.status == "ok" do
site.imported_data
end
dates = dates || %{start_date: nil, end_date: nil}
if site.imported_data && site.imported_data.status == "ok" do
start_date =
[dates.start_date, site.imported_data.start_date]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
[legacy_import, first_import]
|> Enum.reject(&is_nil/1)
|> Enum.min_by(& &1.start_date, Date, fn -> nil end)
end_date =
[dates.end_date, site.imported_data.end_date]
|> Enum.reject(&is_nil/1)
|> Enum.max(Date, fn -> nil end)
%{start_date: start_date, end_date: end_date}
else
dates
end
end
@spec delete_imports_for_site(Site.t()) :: :ok
@ -123,4 +147,66 @@ defmodule Plausible.Imported do
:ok
end
@spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) ::
{:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window}
def check_dates(_site, nil, _end_date), do: {:error, :no_data}
def check_dates(site, start_date, end_date) do
cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
end_date = Enum.min([end_date, cutoff_date], Date)
with true <- Date.diff(end_date, start_date) >= 2,
[_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do
longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first))
{:ok, longest.first, longest.last}
else
_ -> {:error, :no_time_window}
end
end
defp find_free_ranges(start_date, end_date, site) do
occupied_ranges =
site
|> Imported.list_all_imports(Imported.SiteImport.completed())
|> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2))
|> Enum.map(&Date.range(&1.start_date, &1.end_date))
Date.range(start_date, end_date)
|> free_ranges(start_date, occupied_ranges, [])
end
# This function recursively finds open ranges that are not yet occupied
# by existing imported data. The idea is that we keep moving a dynamic
# date index `d` from start until the end of `imported_range`, hopping
# over each occupied range, and capturing the open ranges step-by-step
# in the `result` array.
defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do
cond do
Date.diff(occupied_range.last, d) <= 0 ->
free_ranges(import_range, d, rest_of_occupied_ranges, result)
in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 ->
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
true ->
free_range = Date.range(d, occupied_range.first)
result = result ++ [free_range]
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
end
end
defp free_ranges(import_range, d, [], result) do
if Date.diff(import_range.last, d) < 2 do
result
else
result ++ [Date.range(d, import_range.last)]
end
end
defp in_range?(date, range) do
Date.before?(range.first, date) && Date.after?(range.last, date)
end
end

@ -8,9 +8,11 @@ defmodule Plausible.Imported.Browser do
field :import_id, Ch, type: "UInt64"
field :date, :date
field :browser, :string
field :browser_version, :string
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -43,9 +43,17 @@ defmodule Plausible.Imported.CSVImporter do
{table, _, _} = parse_filename!(filename)
s3_structure = input_structure!(table)
s3_structure_cols_expr =
s3_structure
|> String.split(",", trim: true)
|> Enum.map_join(", ", fn kv ->
[col, _type] = String.split(kv)
col
end)
statement =
"""
INSERT INTO {table:Identifier} \
INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \
SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \
FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \
WHERE date >= {start_date:Date} AND date <= {end_date:Date}\
@ -75,20 +83,21 @@ defmodule Plausible.Imported.CSVImporter do
input_structures = %{
"imported_browsers" =>
"date Date, browser String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_devices" =>
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_entry_pages" =>
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32",
"imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64",
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_exit_pages" =>
"date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
"imported_locations" =>
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_operating_systems" =>
"date Date, operating_system String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_pages" =>
"date Date, hostname String, page String, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
"date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
"imported_sources" =>
"date Date, source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" =>
"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
}

@ -11,6 +11,7 @@ defmodule Plausible.Imported.Device do
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -11,6 +11,7 @@ defmodule Plausible.Imported.EntryPage do
field :visitors, Ch, type: "UInt64"
field :entrances, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -8,7 +8,10 @@ defmodule Plausible.Imported.ExitPage do
field :import_id, Ch, type: "UInt64"
field :date, :date
field :exit_page, :string
field :visitors, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -96,17 +96,28 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
end
defp new_from_report(site_id, import_id, "imported_sources", row) do
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
referrer =
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
end
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
referrer: referrer,
# Only `source` exists in GA4 API
utm_source: nil,
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
@ -120,6 +131,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently
exits: 0,
@ -136,6 +148,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
}
end
@ -148,7 +161,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
# date: get_date(row),
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number()
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
# }
# end
@ -166,6 +182,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
city: city_data && city_data.id,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
@ -179,6 +196,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
@ -202,8 +220,11 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id,
date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser),
# Does not exist in GA4 API
browser_version: nil,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
@ -223,8 +244,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id,
date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os),
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}

@ -13,6 +13,7 @@ defmodule Plausible.Imported.Location do
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -8,9 +8,11 @@ defmodule Plausible.Imported.OperatingSystem do
field :import_id, Ch, type: "UInt64"
field :date, :date
field :operating_system, :string
field :operating_system_version, :string
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -9,6 +9,7 @@ defmodule Plausible.Imported.Page do
field :date, :date
field :hostname, :string
field :page, :string
field :visits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"

@ -8,6 +8,8 @@ defmodule Plausible.Imported.Source do
field :import_id, Ch, type: "UInt64"
field :date, :date
field :source, :string
field :referrer, :string
field :utm_source, :string
field :utm_medium, :string
field :utm_campaign, :string
field :utm_content, :string
@ -15,6 +17,7 @@ defmodule Plausible.Imported.Source do
field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end
end

@ -44,26 +44,6 @@ defmodule Plausible.Ingestion.Event do
changeset: %Ecto.Changeset{}
}
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do
processed_events =
@ -359,8 +339,7 @@ defmodule Plausible.Ingestion.Event do
%{
event
| clickhouse_event:
Map.merge(event.clickhouse_event, Map.take(session, @session_properties))
| clickhouse_event: ClickhouseEventV2.merge_session(event.clickhouse_event, session)
}
end

@ -47,7 +47,14 @@ defmodule Plausible.Session.CacheStore do
do: event.pathname,
else: session.entry_page
),
hostname:
if(event.name == "pageview" and session.hostname == "",
do: event.hostname,
else: session.hostname
),
exit_page: if(event.name == "pageview", do: event.pathname, else: session.exit_page),
exit_page_hostname:
if(event.name == "pageview", do: event.hostname, else: session.exit_page_hostname),
is_bounce: false,
duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews:
@ -60,11 +67,12 @@ defmodule Plausible.Session.CacheStore do
%Plausible.ClickhouseSessionV2{
sign: 1,
session_id: Plausible.ClickhouseSessionV2.random_uint64(),
hostname: event.hostname,
hostname: if(event.name == "pageview", do: event.hostname, else: ""),
site_id: event.site_id,
user_id: event.user_id,
entry_page: if(event.name == "pageview", do: event.pathname, else: ""),
exit_page: if(event.name == "pageview", do: event.pathname, else: ""),
exit_page_hostname: if(event.name == "pageview", do: event.hostname, else: ""),
is_bounce: true,
duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0),

@ -59,7 +59,7 @@ defmodule Plausible.Site do
# `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true
field :earliest_import_end_date, :date, virtual: true
field :latest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps()

@ -5,19 +5,6 @@ defmodule Plausible.Stats.Aggregate do
import Ecto.Query
alias Plausible.Stats.{Query, Util}
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [
:visitors,
:pageviews,
:events,
:sample_percent,
:conversion_rate,
:total_visitors
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
def aggregate(site, query, metrics) do
{currency, metrics} =
on_full_build do
@ -28,18 +15,17 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics)
event_metrics =
{event_metrics, session_metrics, other_metrics} =
metrics
|> Util.maybe_add_visitors_metric()
|> Enum.filter(&(&1 in @event_metrics))
|> Plausible.Stats.TableDecider.partition_metrics(query)
event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task =
if :time_on_page in metrics do
if :time_on_page in other_metrics do
fn -> aggregate_time_on_page(site, query) end
else
fn -> %{} end

@ -3,7 +3,7 @@ defmodule Plausible.Stats.Base do
use Plausible
use Plausible.Stats.Fragments
alias Plausible.Stats.{Query, Filters}
alias Plausible.Stats.{Query, Filters, TableDecider}
alias Plausible.Timezones
import Ecto.Query
@ -15,7 +15,7 @@ defmodule Plausible.Stats.Base do
def base_event_query(site, query) do
events_q = query_events(site, query)
if Enum.any?(Filters.visit_props(), &query.filters["visit:" <> &1]) do
if TableDecider.events_join_sessions?(query) do
sessions_q =
from(
s in query_sessions(site, query),
@ -30,11 +30,15 @@ defmodule Plausible.Stats.Base do
on: e.session_id == sq.session_id
)
else
events_q
if query.experimental_reduced_joins? do
events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)
else
events_q
end
end
end
def query_events(site, query) do
defp query_events(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site)
q =
@ -48,7 +52,10 @@ defmodule Plausible.Stats.Base do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname))
q =
q
|> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))
|> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))
q =
case query.filters["event:name"] do
@ -122,17 +129,6 @@ defmodule Plausible.Stats.Base do
q
end
@api_prop_name_to_db %{
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id"
}
def query_sessions(site, query) do
{first_datetime, last_datetime} =
utc_boundaries(query, site)
@ -140,8 +136,7 @@ defmodule Plausible.Stats.Base do
q = from(s in "sessions_v2", where: s.site_id == ^site.id)
sessions_q =
if FunWithFlags.enabled?(:experimental_session_count, for: site) or
query.experimental_session_count? do
if query.experimental_session_count? do
from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
else
from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime
@ -151,9 +146,24 @@ defmodule Plausible.Stats.Base do
sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
end
sessions_q = filter_by_entry_props(sessions_q, query)
filter_by_entry_props(sessions_q, query)
|> filter_by_visit_props(Filters.visit_props(), query)
end
Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->
@api_prop_name_to_db %{
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id",
"entry_page_hostname" => "hostname"
}
defp filter_by_visit_props(q, visit_props, query) do
Enum.reduce(visit_props, q, fn prop_name, sessions_q ->
filter_key = "visit:" <> prop_name
db_field =
@ -237,6 +247,13 @@ defmodule Plausible.Stats.Base do
}
end
defp select_event_metric(:visits) do
%{
visits:
dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))
}
end
on_full_build do
defp select_event_metric(:total_revenue) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}

@ -6,7 +6,7 @@ defmodule Plausible.Stats.Breakdown do
import Plausible.Stats.{Base, Imported}
import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util}
alias Plausible.Stats.{Query, Util, TableDecider}
@no_ref "Direct / None"
@not_set "(not set)"
@ -103,7 +103,6 @@ defmodule Plausible.Stats.Breakdown do
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
# :TODO: Handle other orderings
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
@ -200,9 +199,35 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
breakdown_sessions(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
case breakdown_table(query, metrics, property) do
:session ->
breakdown_sessions(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
:event ->
breakdown_events(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end
end
# Backwards compatibility
# defp breakdown_table(%Query{experimental_reduced_joins?: false}, _, _), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page_hostname"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page_hostname"), do: :session
defp breakdown_table(query, metrics, property) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, property)
if not Enum.empty?(session_metrics) do
:session
else
:event
end
end
defp zip_results(event_result, session_result, property, metrics) do
@ -273,94 +298,6 @@ defmodule Plausible.Stats.Breakdown do
end
defp breakdown_time_on_page(site, query, pages) do
if FunWithFlags.enabled?(:window_time_on_page) do
window_breakdown_time_on_page(site, query, pages)
else
neighbor_breakdown_time_on_page(site, query, pages)
end
end
defp neighbor_breakdown_time_on_page(site, query, pages) do
q =
from(
e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
select: {
fragment("? as p", e.pathname),
fragment("? as t", e.timestamp),
fragment("? as s", e.session_id)
},
order_by: [e.session_id, e.timestamp]
)
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
select =
if query.include_imported do
"sum(td), count(case when p2 != p then 1 end)"
else
"round(sum(td)/count(case when p2 != p then 1 end))"
end
pages_idx = length(base_query_raw_params)
params = base_query_raw_params ++ [pages]
time_query = "
SELECT
p,
#{select}
FROM
(SELECT
p,
p2,
sum(t2-t) as td
FROM
(SELECT
*,
neighbor(t, 1) as t2,
neighbor(p, 1) as p2,
neighbor(s, 1) as s2
FROM (#{base_query_raw}))
WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}
GROUP BY p,p2,s)
GROUP BY p"
{:ok, res} = ClickhouseRepo.query(time_query, params)
if query.include_imported do
# Imported page views have pre-calculated values
res =
res.rows
|> Enum.map(fn [page, time, visits] -> {page, {time, visits}} end)
|> Enum.into(%{})
from(
i in "imported_pages",
group_by: i.page,
where: i.site_id == ^site.id,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.page in ^pages,
select: %{
page: i.page,
pageviews: fragment("sum(?) - sum(?)", i.pageviews, i.exits),
time_on_page: sum(i.time_on_page)
}
)
|> ClickhouseRepo.all()
|> Enum.reduce(res, fn %{page: page, pageviews: pageviews, time_on_page: time}, res ->
{restime, resviews} = Map.get(res, page, {0, 0})
Map.put(res, page, {restime + time, resviews + pageviews})
end)
|> Enum.map(fn
{page, {_, 0}} -> {page, nil}
{page, {time, pageviews}} -> {page, time / pageviews}
end)
|> Enum.into(%{})
else
res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{})
end
end
defp window_breakdown_time_on_page(site, query, pages) do
import Ecto.Query
windowed_pages_q =

@ -208,11 +208,12 @@ defmodule Plausible.Stats.FilterSuggestions do
"operating_system" -> :operating_system
"operating_system_version" -> :operating_system_version
"screen_size" -> :screen_size
"hostname" -> :hostname
_ -> :unknown
end
q =
if(filter_name == :pathname,
if(filter_name == :pathname or filter_name == :hostname,
do: base_event_query(site, query),
else: query_sessions(site, query)
)
@ -230,6 +231,12 @@ defmodule Plausible.Stats.FilterSuggestions do
where: fragment("? ilike ?", e.pathname, ^filter_query)
)
:hostname ->
from(e in q,
select: e.hostname,
where: fragment("? ilike ?", e.hostname, ^filter_query)
)
:entry_page ->
from(e in q,
select: e.entry_page,

@ -31,7 +31,7 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do
{is_negated, val} = parse_negated_prefix(val)
{is_contains, val} = parse_contains_prefix(val)
is_list = list_expression?(val)
is_wildcard = String.contains?(key, ["page", "goal"]) && wildcard_expression?(val)
is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val)
val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
val = if key == "goal", do: wrap_goal_value(val), else: val

@ -23,11 +23,22 @@ defmodule Plausible.Stats.Filters do
:region,
:city,
:entry_page,
:exit_page
:exit_page,
:entry_page_hostname,
:exit_page_hostname
]
def visit_props(), do: @visit_props |> Enum.map(&to_string/1)
@event_props [:name, :page, :goal]
@event_table_visit_props @visit_props --
[
:entry_page,
:exit_page,
:entry_page_hostname,
:exit_page_hostname
]
def event_table_visit_props(), do: @event_table_visit_props |> Enum.map(&to_string/1)
@event_props [:name, :page, :goal, :hostname]
def event_props(), do: @event_props |> Enum.map(&to_string/1)

@ -9,7 +9,8 @@ defmodule Plausible.Stats.Query do
imported_data_requested: false,
include_imported: false,
now: nil,
experimental_session_count?: false
experimental_session_count?: false,
experimental_reduced_joins?: false
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval}
@ -22,7 +23,8 @@ defmodule Plausible.Stats.Query do
query =
__MODULE__
|> struct!(now: now)
|> put_experimental_session_count(params)
|> put_experimental_session_count(site, params)
|> put_experimental_reduced_joins(site, params)
|> put_period(site, params)
|> put_interval(params)
|> put_parsed_filters(params)
@ -36,11 +38,27 @@ defmodule Plausible.Stats.Query do
query
end
defp put_experimental_session_count(query, params) do
if Map.get(params, "experimental_session_count") == "true" do
struct!(query, experimental_session_count?: true)
defp put_experimental_session_count(query, site, params) do
if Map.has_key?(params, "experimental_session_count") do
struct!(query,
experimental_session_count?: Map.get(params, "experimental_session_count") == "true"
)
else
query
struct!(query,
experimental_session_count?: FunWithFlags.enabled?(:experimental_session_count, for: site)
)
end
end
defp put_experimental_reduced_joins(query, site, params) do
if Map.has_key?(params, "experimental_reduced_joins") do
struct!(query,
experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"
)
else
struct!(query,
experimental_reduced_joins?: FunWithFlags.enabled?(:experimental_reduced_joins, for: site)
)
end
end
@ -261,8 +279,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do
cond do
is_nil(site.earliest_import_end_date) -> false
Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
is_nil(site.latest_import_end_date) -> false
Date.after?(query.date_range.first, site.latest_import_end_date) -> false
Enum.any?(query.filters) -> false
query.period == "realtime" -> false
true -> requested?

@ -0,0 +1,111 @@
defmodule Plausible.Stats.TableDecider do
@moduledoc """
This module contains logic for deciding which tables need to be queried given a query
and metrics, with the purpose of reducing the number of queries and JOINs needed to perform.
"""
import Enum, only: [empty?: 1]
alias Plausible.Stats.Query
def events_join_sessions?(query) do
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
end
def partition_metrics(metrics, query, breakdown_property \\ nil) do
%{
event: event_only_metrics,
session: session_only_metrics,
either: either_metrics,
other: other_metrics,
sample_percent: sample_percent
} =
partition(metrics, query, &metric_partitioner/2)
# Treat breakdown property as yet another filter
filters =
if breakdown_property do
Map.put(query.filters, breakdown_property, nil)
else
query.filters
end
%{event: event_only_filters, session: session_only_filters} =
partition(filters, query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
empty?(event_only_metrics) && empty?(event_only_filters) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) && empty?(session_only_filters) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Filters on both events and sessions, but only one kind of metric
empty?(event_only_metrics) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Default: prefer sessions
true ->
{event_only_metrics ++ sample_percent,
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
end
end
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
defp metric_partitioner(_, :total_revenue), do: :event
defp metric_partitioner(_, :pageviews), do: :event
defp metric_partitioner(_, :events), do: :event
defp metric_partitioner(_, :bounce_rate), do: :session
defp metric_partitioner(_, :visit_duration), do: :session
defp metric_partitioner(_, :views_per_visit), do: :session
# Metrics which used to only be queried from one table but can be calculated from either
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visits), do: :either
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visitors), do: :either
defp metric_partitioner(_, :visits), do: :session
defp metric_partitioner(_, :visitors), do: :event
# Calculated metrics - handled on callsite separately from other metrics.
defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other
defp metric_partitioner(_, :percentage), do: :other
# Sample percentage is included in both tables if queried.
defp metric_partitioner(_, :sample_percent), do: :sample_percent
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
end
defp metric_partitioner(_, _), do: :either
defp filters_partitioner(_, {"event:" <> _, _}), do: :event
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
do: :either
defp filters_partitioner(_, {"visit:" <> _, _}),
do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
end
defp filters_partitioner(_, _), do: :either
@default %{event: [], session: [], either: [], other: [], sample_percent: []}
defp partition(values, query, partitioner) do
Enum.reduce(values, @default, fn value, acc ->
key = partitioner.(query, value)
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
end)
end
end

@ -18,15 +18,11 @@ defmodule Plausible.Stats.Timeseries do
@typep value :: nil | integer() | float()
@type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()})
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [:visitors, :pageviews, :events, :conversion_rate] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit]
def timeseries(site, query, metrics) do
steps = buckets(query)
event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
{event_metrics, session_metrics, _} =
Plausible.Stats.TableDecider.partition_metrics(metrics, query)
{currency, event_metrics} =
on_full_build do

@ -4,7 +4,6 @@ defmodule PlausibleWeb.Api.StatsController do
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Imported.SiteImport
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
@ -68,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController do
* `with_imported` - boolean indicating whether the Google Analytics data
was queried or not.
* `imported_source` - the source of the imported data, when applicable.
Currently only Google Analytics is supported.
* `imports_exist` - boolean indicating whether there are any completed
imports for a given site or not.
* `full_intervals` - map of dates indicating whether the interval has been
cut off by the requested date range or not. For example, if looking at a
@ -87,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController do
"2021-11-01" => true,
"2021-12-01" => false
},
"imported_source" => nil,
"imports_exist" => false,
"interval" => "month",
"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
"plot" => [0, 0, 0, 0],
@ -129,9 +128,8 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
metric: metric,
plot: plot_timeseries(timeseries_result, metric),
labels: labels,
comparison_plot: comparison_result && plot_timeseries(comparison_result, metric),
@ -139,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: with_imported?(query, comparison_query),
imported_source: site_import && SiteImport.label(site_import),
imports_exist: site.complete_import_ids != [],
full_intervals: full_intervals
})
else
@ -214,14 +212,12 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query),
imported_source: site_import && SiteImport.label(site_import),
imports_exist: site.complete_import_ids != [],
comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first,
@ -299,10 +295,12 @@ defmodule PlausibleWeb.Api.StatsController do
},
%{
name: "Unique conversions (last 30 min)",
graph_metric: :visitors,
value: unique_conversions
},
%{
name: "Total conversions (last 30 min)",
graph_metric: :events,
value: total_conversions
}
]
@ -325,10 +323,12 @@ defmodule PlausibleWeb.Api.StatsController do
},
%{
name: "Unique visitors (last 30 min)",
graph_metric: :visitors,
value: visitors
},
%{
name: "Pageviews (last 30 min)",
graph_metric: :pageviews,
value: pageviews
}
]
@ -345,15 +345,21 @@ defmodule PlausibleWeb.Api.StatsController do
[
top_stats_entry(results, comparison, "Unique visitors", :total_visitors),
top_stats_entry(results, comparison, "Unique conversions", :visitors),
top_stats_entry(results, comparison, "Total conversions", :events),
top_stats_entry(results, comparison, "Unique conversions", :visitors, graphable?: true),
top_stats_entry(results, comparison, "Total conversions", :events, graphable?: true),
on_full_build do
top_stats_entry(results, comparison, "Average revenue", :average_revenue, &format_money/1)
top_stats_entry(results, comparison, "Average revenue", :average_revenue,
formatter: &format_money/1,
graphable?: true
)
end,
on_full_build do
top_stats_entry(results, comparison, "Total revenue", :total_revenue, &format_money/1)
top_stats_entry(results, comparison, "Total revenue", :total_revenue,
formatter: &format_money/1,
graphable?: true
)
end,
top_stats_entry(results, comparison, "Conversion rate", :conversion_rate)
top_stats_entry(results, comparison, "Conversion rate", :conversion_rate, graphable?: true)
]
|> Enum.reject(&is_nil/1)
|> then(&{&1, 100})
@ -387,39 +393,63 @@ defmodule PlausibleWeb.Api.StatsController do
stats =
[
top_stats_entry(current_results, prev_results, "Unique visitors", :visitors),
top_stats_entry(current_results, prev_results, "Total visits", :visits),
top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews),
top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit),
top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate),
top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration),
top_stats_entry(current_results, prev_results, "Time on page", :time_on_page, fn
nil -> 0
value -> value
end)
top_stats_entry(current_results, prev_results, "Unique visitors", :visitors,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Total visits", :visits, graphable?: true),
top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Time on page", :time_on_page,
formatter: fn
nil -> 0
value -> value
end
)
]
|> Enum.filter(& &1)
{stats, current_results[:sample_percent][:value]}
end
defp top_stats_entry(current_results, prev_results, name, key, formatter \\ & &1) do
defp top_stats_entry(current_results, prev_results, name, key, opts \\ []) do
if current_results[key] do
formatter = Keyword.get(opts, :formatter, & &1)
value = get_in(current_results, [key, :value])
if prev_results do
prev_value = get_in(prev_results, [key, :value])
change = Stats.Compare.calculate_change(key, prev_value, value)
%{name: name, value: formatter.(value)}
|> maybe_put_graph_metric(opts, key)
|> maybe_put_comparison(prev_results, key, value, formatter)
end
end
%{
name: name,
value: formatter.(value),
comparison_value: formatter.(prev_value),
change: change
}
else
%{name: name, value: formatter.(value)}
end
defp maybe_put_graph_metric(entry, opts, key) do
if Keyword.get(opts, :graphable?) do
entry |> Map.put(:graph_metric, key)
else
entry
end
end
defp maybe_put_comparison(entry, prev_results, key, value, formatter) do
if prev_results do
prev_value = get_in(prev_results, [key, :value])
change = Stats.Compare.calculate_change(key, prev_value, value)
entry
|> Map.put(:comparison_value, formatter.(prev_value))
|> Map.put(:change, change)
else
entry
end
end
@ -428,6 +458,13 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.from(site, params)
pagination = parse_pagination(params)
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:entry_page_hostname", query.filters["event:hostname"])
else
query
end
extra_metrics =
if params["detailed"], do: [:bounce_rate, :visit_duration], else: []
@ -733,6 +770,13 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits, :visit_duration])
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:entry_page_hostname", query.filters["event:hostname"])
else
query
end
entry_pages =
Stats.breakdown(site, query, "visit:entry_page", metrics, pagination)
|> transform_keys(%{entry_page: :name})
@ -763,6 +807,13 @@ defmodule PlausibleWeb.Api.StatsController do
{limit, page} = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits])
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:exit_page_hostname", query.filters["event:hostname"])
else
query
end
exit_pages =
Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page})
|> add_exit_rate(site, query, limit)

@ -1,6 +1,11 @@
defmodule PlausibleWeb.GoogleAnalyticsController do
use PlausibleWeb, :controller
alias Plausible.Google
alias Plausible.Imported
require Plausible.Imported.SiteImport
plug(PlausibleWeb.RequireAccountPlug)
plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
@ -10,6 +15,8 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"start_date" => start_date,
"end_date" => end_date,
"legacy" => legacy
}) do
site = conn.assigns.site
@ -22,17 +29,22 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
start_date: start_date,
end_date: end_date,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def property_or_view_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
def property_or_view_form(
conn,
%{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
} = params
) do
site = conn.assigns.site
redirect_route =
@ -44,9 +56,21 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
result =
if legacy == "true" do
Plausible.Google.UA.API.list_views(access_token)
Google.UA.API.list_views(access_token)
else
Plausible.Google.API.list_properties_and_views(access_token)
Google.API.list_properties_and_views(access_token)
end
error =
case params["error"] do
"no_data" ->
"No data found. Nothing to import."
"no_time_window" ->
"Imported data time range is completely overlapping with existing data. Nothing to import."
_ ->
nil
end
case result do
@ -59,6 +83,7 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
expires_at: expires_at,
site: conn.assigns.site,
properties_and_views: properties_and_views,
selected_property_or_view_error: error,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
@ -75,65 +100,81 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
conn
|> put_flash(
:error,
"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
"We were unable to list your Google Analytics properties and views. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
# see https://stackoverflow.com/a/57416769
@google_analytics_new_user_metric_date ~D[2016-08-24]
@universal_analytics_new_user_metric_date ~D[2016-08-24]
def property_or_view(conn, %{
"property_or_view" => property_or_view,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
def property_or_view(
conn,
%{
"property_or_view" => property_or_view,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
} = params
) do
site = conn.assigns.site
start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view)
case start_date do
{:ok, nil} ->
{:ok, properties_and_views} =
if legacy == "true" do
Plausible.Google.UA.API.list_views(access_token)
else
Plausible.Google.API.list_properties_and_views(access_token)
end
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
else
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
with {:ok, api_start_date} <-
Google.API.get_analytics_start_date(access_token, property_or_view),
{:ok, api_end_date} <- Google.API.get_analytics_end_date(access_token, property_or_view),
{:ok, start_date, end_date} <- Imported.check_dates(site, api_start_date, api_end_date) do
action =
if Timex.before?(api_start_date, @universal_analytics_new_user_metric_date) do
:user_metric_notice
else
:confirm
end
redirect(conn,
external:
Routes.google_analytics_path(conn, action, site.domain,
property_or_view: property_or_view,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
start_date: Date.to_iso8601(start_date),
end_date: Date.to_iso8601(end_date),
legacy: legacy
)
)
else
{:error, error} when error in [:no_data, :no_time_window] ->
params =
params
|> Map.take(["access_token", "refresh_token", "expires_at", "legacy"])
|> Map.put("error", Atom.to_string(error))
property_or_view_form(conn, params)
{:error, :authentication_failed} ->
conn
|> assign(:skip_plausible_tracking, true)
|> render("property_or_view_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
properties_and_views: properties_and_views,
selected_property_or_view_error: "No data found. Nothing to import",
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
|> put_flash(
:error,
"Google Analytics authentication seems to have expired. Please try again."
)
|> redirect(external: redirect_route)
{:ok, date} ->
action =
if Timex.before?(date, @google_analytics_new_user_metric_date) do
:user_metric_notice
else
:confirm
end
redirect(conn,
to:
Routes.google_analytics_path(conn, action, site.domain,
property_or_view: property_or_view,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
@ -142,32 +183,64 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"start_date" => start_date,
"end_date" => end_date,
"legacy" => legacy
}) do
site = conn.assigns.site
start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view)
start_date = Date.from_iso8601!(start_date)
end_date = Date.from_iso8601!(end_date)
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
else
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
case Google.API.get_property_or_view(access_token, property_or_view) do
{:ok, %{name: property_or_view_name, id: property_or_view}} ->
conn
|> assign(:skip_plausible_tracking, true)
|> render("confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_property_or_view: property_or_view,
selected_property_or_view_name: property_or_view_name,
start_date: start_date,
end_date: end_date,
property?: Google.API.property?(property_or_view),
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
{:error, :authentication_failed} ->
conn
|> put_flash(
:error,
"Google Analytics authentication seems to have expired. Please try again."
)
|> redirect(external: redirect_route)
{:ok, %{name: property_or_view_name, id: property_or_view}} =
Plausible.Google.API.get_property_or_view(access_token, property_or_view)
{:error, :not_found} ->
conn
|> put_flash(
:error,
"Google Analytics property not found. Please try again."
)
|> redirect(external: redirect_route)
conn
|> assign(:skip_plausible_tracking, true)
|> render("confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_property_or_view: property_or_view,
selected_property_or_view_name: property_or_view_name,
start_date: start_date,
end_date: end_date,
property?: Plausible.Google.API.property?(property_or_view),
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
def import(conn, %{
@ -182,6 +255,9 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
site = conn.assigns.site
current_user = conn.assigns.current_user
start_date = Date.from_iso8601!(start_date)
end_date = Date.from_iso8601!(end_date)
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
@ -189,36 +265,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
if Plausible.Google.API.property?(property_or_view) do
{:ok, _} =
Plausible.Imported.GoogleAnalytics4.new_import(
site,
current_user,
property: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at
case Imported.check_dates(site, start_date, end_date) do
{:ok, start_date, end_date} ->
if Google.API.property?(property_or_view) do
{:ok, _} =
Imported.GoogleAnalytics4.new_import(
site,
current_user,
property: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at
)
else
{:ok, _} =
Imported.UniversalAnalytics.new_import(
site,
current_user,
view_id: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
end
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
{:error, :no_time_window} ->
conn
|> put_flash(
:error,
"Import failed. No data could be imported because date range overlaps with existing data."
)
else
Plausible.Imported.UniversalAnalytics.new_import(
site,
current_user,
view_id: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
|> redirect(external: redirect_route)
end
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
end
end

@ -666,6 +666,8 @@ defmodule PlausibleWeb.SiteController do
Plausible.Repo.delete!(site_import)
if site_import.legacy do
Plausible.Purge.delete_imported_stats!(site, 0)
site
|> Plausible.Site.remove_imported_data()
|> Repo.update!()

@ -64,13 +64,14 @@ defmodule PlausibleWeb.StatsController do
|> render("stats.html",
site: site,
has_goals: Plausible.Sites.has_goals?(site),
revenue_goals: list_revenue_goals(site),
funnels: list_funnels(site),
has_props: Plausible.Props.configured?(site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
title: title(conn, site),
demo: demo,
flags: get_flags(conn.assigns[:current_user]),
flags: get_flags(conn.assigns[:current_user], site),
is_dbip: is_dbip(),
dogfood_page_path: dogfood_page_path,
load_dashboard_js: true
@ -92,10 +93,13 @@ defmodule PlausibleWeb.StatsController do
defp list_funnels(site) do
Plausible.Funnels.list(site)
end
else
defp list_funnels(_site) do
[]
defp list_revenue_goals(site) do
Plausible.Goals.list_revenue_goals(site)
end
else
defp list_funnels(_site), do: []
defp list_revenue_goals(_site), do: []
end
@doc """
@ -319,6 +323,7 @@ defmodule PlausibleWeb.StatsController do
|> render("stats.html",
site: shared_link.site,
has_goals: Sites.has_goals?(shared_link.site),
revenue_goals: list_revenue_goals(shared_link.site),
funnels: list_funnels(shared_link.site),
has_props: Plausible.Props.configured?(shared_link.site),
stats_start_date: stats_start_date,
@ -330,7 +335,7 @@ defmodule PlausibleWeb.StatsController do
embedded: conn.params["embed"] == "true",
background: conn.params["background"],
theme: conn.params["theme"],
flags: get_flags(conn.assigns[:current_user]),
flags: get_flags(conn.assigns[:current_user], shared_link.site),
is_dbip: is_dbip(),
load_dashboard_js: true
)
@ -348,8 +353,12 @@ defmodule PlausibleWeb.StatsController do
defp shared_link_cookie_name(slug), do: "shared-link-" <> slug
defp get_flags(_user) do
%{}
defp get_flags(user, site) do
%{
hostname_filter:
FunWithFlags.enabled?(:hostname_filter, for: user) ||
FunWithFlags.enabled?(:hostname_filter, for: site)
}
end
defp is_dbip() do

@ -62,9 +62,11 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
<li :for={entry <- @site_imports} class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %>
<%= Plausible.Imported.SiteImport.label(entry.site_import) %>
<span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">
(<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views)
(<%= PlausibleWeb.StatsView.large_number_format(
Map.get(@pageview_counts, entry.site_import.id, 0)
) %> page views)
</span>
<Heroicons.clock
:if={entry.live_status == SiteImport.pending()}

@ -6,22 +6,8 @@
<%= link("Click here", to: @link) %> to view your dashboard.
<% else %>
Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
<br /> <br />
This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
<br /> <br />
Please log into your Google Analytics account and go to Admin -> Property Settings -> User Analysis. Then make sure that "Enable User Metric in Reporting" is set to OFF. Then try importing again. You can switch the setting back after the import.
<br /> <br /> If you've already changed that setting, then it is something else: <br />
<ul>
<li>
Do you have any stats in the Google Analytics property you tried to import? We only import the stats from your first Google Analytics visitor until your first Plausible Analytics visitor. The import fails if there's no returned data from Google from before you counted your first Plausible visitor.
</li>
<li>
Did you set strict data retention limits in your Google Analytics account? You can check the data retention limit you have set for your property in your Google Analytics admin settings by visiting the "Tracking Info" section and clicking on "Data Retention".
</li>
</ul>
<br />
If all of the above is fine, please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<br /><br />
Please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<%= if full_build?() do %>
<br /> <br />
Please reply to this email to let us know if you're still experiencing issues with the import.

@ -6,55 +6,47 @@
<%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= hidden_input(f, :legacy, value: @legacy) %>
<%= case @start_date do %>
<% {:ok, start_date} -> %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Stats from this
<%= if @property? do %>
property
<% else %>
view
<% end %>
and time period will be imported from your Google Analytics account to your Plausible dashboard
</div>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Stats from this
<%= if @property? do %>
property
<% else %>
view
<% end %>
and time period will be imported from your Google Analytics account to your Plausible dashboard
</div>
<div class="mt-6">
<%= styled_label(
f,
:property_or_view,
"Google Analytics #{if @property?, do: "property", else: "view"}"
) %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= @selected_property_or_view_name %>
</span>
<%= hidden_input(f, :property_or_view,
readonly: "true",
value: @selected_property_or_view
) %>
</div>
<div class="flex justify-between mt-3">
<div class="w-36">
<%= styled_label(f, :start_date, "From") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(start_date) %>
</span>
<%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
</div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36">
<%= styled_label(f, :end_date, "To") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@end_date) %>
</span>
<%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div>
</div>
<% {:error, error} -> %>
<p class="text-gray-700 dark:text-gray-300 mt-6">
The following error occurred when fetching your Google Analytics data.
</p>
<p class="text-red-700 font-medium mt-3"><%= error %></p>
<% end %>
<div class="mt-6">
<%= styled_label(
f,
:property_or_view,
"Google Analytics #{if @property?, do: "property", else: "view"}"
) %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= @selected_property_or_view_name %>
</span>
<%= hidden_input(f, :property_or_view,
readonly: "true",
value: @selected_property_or_view
) %>
</div>
<div class="flex justify-between mt-3">
<div class="w-36">
<%= styled_label(f, :start_date, "From") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@start_date) %>
</span>
<%= hidden_input(f, :start_date, value: @start_date, readonly: "true") %>
</div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36">
<%= styled_label(f, :end_date, "To") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@end_date) %>
</span>
<%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div>
</div>
<%= submit("Confirm import", class: "button mt-6") %>
<% end %>

@ -39,6 +39,8 @@
access_token: @access_token,
refresh_token: @refresh_token,
expires_at: @expires_at,
start_date: @start_date,
end_date: @end_date,
legacy: @legacy
),
class: "button mt-6"

@ -27,6 +27,7 @@
data-props-available={
to_string(Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok)
}
data-revenue-goals={Jason.encode!(@revenue_goals)}
data-funnels={Jason.encode!(@funnels)}
data-has-props={to_string(@has_props)}
data-logged-in={to_string(!!@conn.assigns[:current_user])}

@ -0,0 +1,35 @@
defmodule Plausible.IngestRepo.Migrations.AddMetricsToImportedTables do
use Ecto.Migration
@add_pageviews_for [
"imported_browsers",
"imported_devices",
"imported_entry_pages",
"imported_exit_pages",
"imported_locations",
"imported_operating_systems",
"imported_sources"
]
def up do
for table <- @add_pageviews_for do
add_column(table, "pageviews", "UInt64")
end
add_column("imported_browsers", "browser_version", "String")
add_column("imported_exit_pages", "bounces", "UInt32")
add_column("imported_exit_pages", "visit_duration", "UInt64")
add_column("imported_operating_systems", "operating_system_version", "String")
add_column("imported_pages", "visits", "UInt64")
add_column("imported_sources", "referrer", "String")
add_column("imported_sources", "utm_source", "String")
end
defp add_column(table, column, type) do
execute """
ALTER TABLE #{table}
#{Plausible.MigrationUtils.on_cluster_statement(table)}
ADD COLUMN #{column} #{type}
"""
end
end

@ -0,0 +1,19 @@
defmodule Plausible.ClickhouseRepo.Migrations.HostnamesInSessions do
use Ecto.Migration
def up do
execute """
ALTER TABLE sessions_v2
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
ADD COLUMN exit_page_hostname String CODEC(ZSTD(3))
"""
end
def down do
execute """
ALTER TABLE sessions_v2
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
DROP COLUMN exit_page_hostname
"""
end
end

@ -12,6 +12,7 @@
FunWithFlags.enable(:imports_exports)
FunWithFlags.enable(:shield_pages)
FunWithFlags.enable(:hostname_filter)
user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible")
@ -161,7 +162,7 @@ native_stats_range
[
site_id: site.id,
hostname: site.domain,
hostname: Enum.random(["en.dummy.site", "es.dummy.site", "dummy.site"]),
timestamp: put_random_time.(date, index),
referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]),
@ -196,7 +197,7 @@ native_stats_range
[
name: goal4.event_name,
site_id: site.id,
hostname: site.domain,
hostname: Enum.random(["en.dummy.site", "es.dummy.site", "dummy.site"]),
timestamp: put_random_time.(date, index),
referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]),

@ -0,0 +1,188 @@
defmodule Plausible.DataMigration.SiteImportsTest do
use Plausible.DataCase, async: true
import ExUnit.CaptureIO
alias Plausible.DataMigration.SiteImports
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
describe "run/1" do
test "runs for empty dataset" do
dry_run_output =
capture_io(fn ->
assert :ok = SiteImports.run()
end)
assert dry_run_output =~ "Processing 0 sites"
assert dry_run_output =~ "DRY RUN: true"
real_run_output =
capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
end)
assert real_run_output =~ "Processing 0 sites"
assert real_run_output =~ "DRY RUN: false"
end
test "adds site import entry when it's missing and adjusts end date" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
build(:imported_visitors, date: ~D[2021-01-07])
])
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
end) =~ "Processing 1 sites"
site = Repo.reload!(site)
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
assert id > 0
assert site_import.start_date == site.imported_data.start_date
assert site_import.end_date == ~D[2021-01-07]
assert site_import.source == :universal_analytics
end
test "runs in dry mode without making any persistent changes" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
build(:imported_visitors, date: ~D[2021-01-07])
])
assert capture_io(fn ->
assert :ok = SiteImports.run()
end) =~ "Processing 1 sites"
assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)
assert id == 0
end
test "does not set end date to latter than the current one" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
populate_stats(site, 0, [
build(:imported_visitors, date: ~D[2021-01-10])
])
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
end) =~ "Processing 1 sites"
site = Repo.reload!(site)
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
assert id > 0
assert site_import.start_date == site.imported_data.start_date
assert site_import.end_date == ~D[2021-01-08]
assert site_import.source == :universal_analytics
end
test "leaves site and imports unchanged if everything fits" do
site =
insert(:site)
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|> Repo.update!()
existing_import =
insert(:site_import,
site: site,
start_date: ~D[2021-01-02],
end_date: ~D[2021-01-08],
status: :completed,
legacy: true
)
populate_stats(site, existing_import.id, [
build(:imported_visitors, date: ~D[2021-01-08])
])
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
end) =~ "Processing 1 sites"
site = Repo.reload!(site)
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
assert id == existing_import.id
assert site_import.start_date == site.imported_data.start_date
assert site_import.end_date == ~D[2021-01-08]
assert site_import.source == :universal_analytics
end
end
describe "imported_stats_end_date/1" do
test "returns nil when there are no stats" do
site_import = insert(:site_import)
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) == nil
end
test "returns date when there are stats recorded in one imported table" do
site_import = insert(:site_import)
populate_stats(site_import.site, site_import.id, [
build(:imported_visitors, date: ~D[2021-01-01])
])
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
~D[2021-01-01]
end
test "returns max date across all imported tables" do
site_import = insert(:site_import)
populate_stats(site_import.site, site_import.id, [
build(:imported_visitors, date: ~D[2021-01-01]),
build(:imported_visitors, date: ~D[2021-01-07]),
build(:imported_sources, date: ~D[2021-01-01]),
build(:imported_sources, date: ~D[2021-01-08]),
build(:imported_entry_pages, date: ~D[2021-01-02]),
build(:imported_entry_pages, date: ~D[2021-02-11]),
build(:imported_exit_pages, date: ~D[2021-01-01]),
build(:imported_exit_pages, date: ~D[2021-01-08]),
build(:imported_locations, date: ~D[2021-01-01]),
build(:imported_locations, date: ~D[2021-01-08]),
build(:imported_devices, date: ~D[2021-01-01]),
build(:imported_devices, date: ~D[2021-01-08]),
build(:imported_browsers, date: ~D[2021-01-01]),
build(:imported_browsers, date: ~D[2021-01-08]),
build(:imported_operating_systems, date: ~D[2021-01-01]),
build(:imported_operating_systems, date: ~D[2021-01-08])
])
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
~D[2021-02-11]
end
test "considers all imported tables" do
date = ~D[2021-01-11]
for {table, idx} <- Enum.with_index(Imported.tables()) do
site_import = insert(:site_import)
end_date = Date.add(date, idx)
populate_stats(site_import.site, site_import.id, [
build(String.to_atom(table), date: date),
build(String.to_atom(table), date: end_date)
])
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
end_date
end
end
end
end

@ -92,6 +92,24 @@ defmodule Plausible.GoalsTest do
assert [currency: {"is invalid", _}] = changeset.errors
end
@tag :full_build_only
test "list_revenue_goals/1 lists event_names and currencies for each revenue goal" do
site = insert(:site)
Goals.create(site, %{"event_name" => "One", "currency" => "EUR"})
Goals.create(site, %{"event_name" => "Two", "currency" => "EUR"})
Goals.create(site, %{"event_name" => "Three", "currency" => "USD"})
Goals.create(site, %{"event_name" => "Four"})
Goals.create(site, %{"page_path" => "/some-page"})
revenue_goals = Goals.list_revenue_goals(site)
assert length(revenue_goals) == 3
assert %{event_name: "One", currency: :EUR} in revenue_goals
assert %{event_name: "Two", currency: :EUR} in revenue_goals
assert %{event_name: "Three", currency: :USD} in revenue_goals
end
test "create/2 clears currency for pageview goals" do
site = insert(:site)
{:ok, goal} = Goals.create(site, %{"page_path" => "/purchase", "currency" => "EUR"})

@ -55,4 +55,17 @@ defmodule Plausible.Google.GA4.APITest do
GA4.API.get_analytics_start_date("some_access_token", "properties/153293282")
end
end
describe "get_analytics_end_date/2" do
test "returns stats end date for a given property" do
result = Jason.decode!(File.read!("fixture/ga4_end_date.json"))
expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body ->
{:ok, %Finch.Response{status: 200, body: result}}
end)
assert {:ok, ~D[2024-03-02]} =
GA4.API.get_analytics_end_date("some_access_token", "properties/153293282")
end
end
end

@ -82,15 +82,15 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
Enum.each(Plausible.Imported.tables(), fn table ->
count =
case table do
"imported_sources" -> 3
"imported_visitors" -> 3
"imported_pages" -> 8
"imported_entry_pages" -> 4
"imported_sources" -> 1090
"imported_visitors" -> 31
"imported_pages" -> 3340
"imported_entry_pages" -> 2934
"imported_exit_pages" -> 0
"imported_locations" -> 4
"imported_devices" -> 4
"imported_browsers" -> 5
"imported_operating_systems" -> 4
"imported_locations" -> 2291
"imported_devices" -> 93
"imported_browsers" -> 233
"imported_operating_systems" -> 1068
end
query = from(imported in table, where: imported.site_id == ^site.id)

@ -59,4 +59,208 @@ defmodule Plausible.ImportedTest do
] = Imported.list_all_imports(site)
end
end
describe "get_imports_date_range/1" do
test "returns empty when there are no imports" do
site = insert(:site)
assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
end
test "returns empty when only incomplete or failed imports are present" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"error"
)
|> Repo.update!()
_import1 = insert(:site_import, site: site, status: :pending)
_import2 = insert(:site_import, site: site, status: :importing)
_import3 = insert(:site_import, site: site, status: :failed)
_rogue_import = insert(:site_import, site: build(:site), status: :completed)
assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
end
test "returns start and end dates considering all imports" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"ok"
)
|> Repo.update!()
_import1 =
insert(:site_import,
site: site,
start_date: ~D[2020-04-02],
end_date: ~D[2022-06-22],
status: :completed,
legacy: true
)
_import2 =
insert(:site_import,
site: site,
start_date: ~D[2022-06-22],
end_date: ~D[2024-01-08],
status: :completed
)
assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =
Imported.get_imports_date_range(site)
end
end
describe "check_dates/3" do
test "crops dates from both ends when overlapping with existing import and native stats" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
start_date = ~D[2016-04-03]
end_date = ~D[2021-05-12]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
assert {:ok, ~D[2021-05-12], ~D[2023-10-25]} =
Imported.check_dates(site, ~D[2021-04-11], ~D[2024-01-12])
end
test "picks longest continuous range when containing existing import" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
start_date = ~D[2019-04-03]
end_date = ~D[2021-05-12]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
assert {:ok, ~D[2021-05-12], ~D[2023-10-25]} =
Imported.check_dates(site, ~D[2019-03-21], ~D[2024-01-12])
end
test "does not alter the dates when there are no imports and no native stats" do
site = insert(:site)
assert {:ok, ~D[2021-05-12], ~D[2024-01-12]} =
Imported.check_dates(site, ~D[2021-05-12], ~D[2024-01-12])
end
test "ignores input date range difference smaller than 2 days" do
site = insert(:site)
assert {:error, :no_time_window} =
Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-12])
assert {:error, :no_time_window} =
Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-13])
assert {:ok, ~D[2024-01-12], ~D[2024-01-14]} =
Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-14])
end
test "ignores imports with date range difference smaller than 2 days" do
site = insert(:site)
start_date = ~D[2024-01-12]
end_date = ~D[2024-01-13]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
assert {:ok, ~D[2021-04-22], ~D[2024-03-14]} =
Imported.check_dates(site, ~D[2021-04-22], ~D[2024-03-14])
end
test "returns no time window when input range starts after native stats start date" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
assert {:error, :no_time_window} =
Imported.check_dates(site, ~D[2023-10-28], ~D[2024-01-13])
end
test "returns no time window when input range starts less than 2 days before native stats start date" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
assert {:error, :no_time_window} =
Imported.check_dates(site, ~D[2023-10-24], ~D[2024-01-13])
end
test "crops time range at native stats start date when effective range is 2 days or longer" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
assert {:ok, ~D[2023-10-23], ~D[2023-10-25]} =
Imported.check_dates(site, ~D[2023-10-23], ~D[2024-01-13])
end
test "returns no data error when start date missing" do
site = insert(:site)
assert {:error, :no_data} = Imported.check_dates(site, nil, nil)
end
test "returns no time window error when date range overlaps with existing import and stats completely" do
site = insert(:site)
populate_stats(site, [
build(:pageview, timestamp: ~N[2023-10-25 15:58:00])
])
start_date = ~D[2016-04-03]
end_date = ~D[2023-10-25]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
assert {:error, :no_time_window} =
Imported.check_dates(site, ~D[2021-04-11], ~D[2024-01-12])
end
end
end

@ -89,6 +89,131 @@ defmodule Plausible.Session.CacheStoreTest do
assert session.events == 2
end
describe "hostname-related attributes" do
test "initial for non-pageview" do
site_id = new_site_id()
event =
build(:event,
name: "custom_event",
site_id: site_id,
pathname: "/path/1",
hostname: "example.com"
)
flush([event])
session = get_session(site_id)
assert session.hostname == ""
assert session.exit_page_hostname == ""
end
test "initial for pageview" do
site_id = new_site_id()
event =
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/path/1",
hostname: "example.com"
)
flush([event])
session = get_session(site_id)
assert session.hostname == "example.com"
assert session.exit_page_hostname == "example.com"
end
test "subsequent pageview after custom_event" do
site_id = new_site_id()
events = [
build(:event,
name: "custom_event",
site_id: site_id,
pathname: "/path/1",
hostname: "whatever.example.com",
timestamp: Timex.shift(Timex.now(), seconds: -5),
user_id: 1
),
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/path/2",
hostname: "example.com",
user_id: 1
)
]
flush(events)
session = get_session(site_id)
assert session.hostname == "example.com"
assert session.exit_page_hostname == "example.com"
end
test "hostname change" do
site_id = new_site_id()
events = [
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/landing",
hostname: "example.com",
timestamp: Timex.shift(Timex.now(), seconds: -5),
user_id: 1
),
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/post/1",
hostname: "blog.example.com",
user_id: 1
)
]
flush(events)
session = get_session(site_id)
assert session.hostname == "example.com"
assert session.exit_page_hostname == "blog.example.com"
end
test "hostname change with custom event in the middle" do
site_id = new_site_id()
events = [
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/landing",
hostname: "example.com",
timestamp: Timex.shift(Timex.now(), seconds: -5),
user_id: 1
),
build(:event,
name: "custom_event",
site_id: site_id,
pathname: "/path/1",
hostname: "analytics.example.com",
timestamp: Timex.shift(Timex.now(), seconds: -3),
user_id: 1
),
build(:event,
name: "pageview",
site_id: site_id,
pathname: "/post/1",
hostname: "blog.example.com",
user_id: 1
)
]
flush(events)
session = get_session(site_id)
assert session.hostname == "example.com"
assert session.exit_page_hostname == "blog.example.com"
end
end
test "initial pageview-specific attributes" do
site_id = new_site_id()
@ -231,7 +356,12 @@ defmodule Plausible.Session.CacheStoreTest do
end
defp get_session(site_id) do
session_q = from s in Plausible.ClickhouseSessionV2, where: s.site_id == ^site_id
session_q =
from s in Plausible.ClickhouseSessionV2,
where: s.site_id == ^site_id,
order_by: [desc: :timestamp],
limit: 1
Plausible.ClickhouseRepo.one!(session_q)
end
end

@ -67,6 +67,9 @@ defmodule Plausible.Stats.DashboardFilterParserTest do
%{"props" => %{"cta" => "Top"}}
|> assert_parsed(%{"event:props:cta" => {:is, "Top"}})
%{"hostname" => "dummy.site"}
|> assert_parsed(%{"event:hostname" => {:is, "dummy.site"}})
end
end

@ -0,0 +1,164 @@
defmodule Plausible.Stats.TableDeciderTest do
use Plausible.DataCase, async: true
alias Plausible.Stats.Query
import Plausible.Stats.TableDecider
test "events_join_sessions? with experimental_reduced_joins disabled" do
assert not events_join_sessions?(make_query(false, %{}))
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
assert events_join_sessions?(make_query(false, %{source: "Google"}))
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
end
test "events_join_sessions? with experimental_reduced_joins enabled" do
assert not events_join_sessions?(make_query(true, %{}))
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
end
describe "partition_metrics" do
test "with no metrics or filters" do
query = make_query(false, %{})
assert partition_metrics([], query) == {[], [], []}
end
test "session-only metrics accordingly" do
query = make_query(false, %{})
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
{[], [:bounce_rate, :views_per_visit], []}
end
test "event-only metrics accordingly" do
query = make_query(false, %{})
assert partition_metrics([:total_revenue, :visitors], query) ==
{[:total_revenue, :visitors], [], []}
end
test "filters from both, event-only metrics" do
query = make_query(false, %{name: "pageview", source: "Google"})
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
end
test "filters from both, session-only metrics" do
query = make_query(false, %{name: "pageview", source: "Google"})
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
end
test "session filters but no session metrics" do
query = make_query(false, %{source: "Google"})
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
end
test "sample_percent is added to both types of metrics" do
query = make_query(false, %{})
assert partition_metrics([:total_revenue, :sample_percent], query) ==
{[:total_revenue, :sample_percent], [], []}
assert partition_metrics([:bounce_rate, :sample_percent], query) ==
{[], [:bounce_rate, :sample_percent], []}
assert partition_metrics([:total_revenue, :bounce_rate, :sample_percent], query) ==
{[:total_revenue, :sample_percent], [:bounce_rate, :sample_percent], []}
end
test "other metrics put in its own result" do
query = make_query(false, %{})
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
{[], [], [:time_on_page, :percentage, :total_visitors]}
end
test "raises if unknown metric" do
query = make_query(false, %{})
assert_raise ArgumentError, fn ->
partition_metrics([:foobar], query)
end
end
end
describe "partition_metrics with experimental_reduced_joins enabled" do
test "metrics that can be calculated on either when event-only metrics" do
query = make_query(true, %{})
assert partition_metrics([:total_revenue, :visitors], query) ==
{[:total_revenue, :visitors], [], []}
assert partition_metrics([:pageviews, :visits], query) == {[:pageviews, :visits], [], []}
end
test "metrics that can be calculated on either when session-only metrics" do
query = make_query(true, %{})
assert partition_metrics([:bounce_rate, :visitors], query) ==
{[], [:bounce_rate, :visitors], []}
assert partition_metrics([:visit_duration, :visits], query) ==
{[], [:visit_duration, :visits], []}
end
test "metrics that can be calculated on either are biased to sessions" do
query = make_query(true, %{})
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
{[:total_revenue], [:bounce_rate, :visitors], []}
end
test "sample_percent is handled with either metrics" do
query = make_query(true, %{})
assert partition_metrics([:visitors, :sample_percent], query) ==
{[], [:visitors, :sample_percent], []}
end
test "metric can be calculated on either, but filtering on events" do
query = make_query(true, %{name: "pageview"})
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
end
test "metric can be calculated on either, but filtering on events and sessions" do
query = make_query(true, %{name: "pageview", exit_page: "/"})
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "metric can be calculated on either, filtering on either" do
query = make_query(true, %{source: "Google"})
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "metric can be calculated on either, filtering on sessions" do
query = make_query(true, %{exit_page: "/"})
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
end
test "breakdown value leans metric" do
query = make_query(true, %{})
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
end
end
defp make_query(experimental_reduced_joins?, filters) do
Query.from(build(:site), %{
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
"filters" => Jason.encode!(filters)
})
end
end

@ -1355,50 +1355,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
assert json_response(conn, 200)["results"] == %{"visitors" => %{"value" => 3}}
end
test "joins correctly with the sessions (CollapsingMergeTree) table", %{
test "handles filtering by visit country", %{
conn: conn,
site: site
} do
create_sessions([
%{
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: 1,
events: 1
},
%{
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: -1,
events: 1
},
%{
site_id: site.id,
session_id: 1000,
country_code: "EE",
sign: 1,
events: 2
}
])
create_events([
%{
site_id: site.id,
session_id: 1000,
name: "pageview"
},
%{
site_id: site.id,
session_id: 1000,
name: "pageview"
},
%{
site_id: site.id,
session_id: 1000,
name: "pageview"
}
populate_stats(site, [
build(:pageview, country_code: "EE"),
build(:pageview, country_code: "EE"),
build(:pageview, country_code: "EE")
])
conn =

@ -918,7 +918,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
timestamp: ~N[2021-01-01 05:00:00]
),
build(:pageview,
pathname: "/goobye",
pathname: "/goodbye",
timestamp: ~N[2021-01-01 00:00:00]
)
])
@ -944,6 +944,50 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
}
end
test "can filter by hostname", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview,
user_id: @user_id,
hostname: "landing.example.com",
timestamp: ~N[2021-01-01 00:00:01]
),
build(:pageview,
user_id: @user_id,
hostname: "example.com",
timestamp: ~N[2021-01-01 00:00:02]
),
build(:pageview,
user_id: @user_id,
hostname: "example.com",
timestamp: ~N[2021-01-01 00:00:06]
)
])
conn =
get(conn, "/api/v1/stats/timeseries", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"filters" => "event:hostname==example.com",
"metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration"
})
res =
json_response(conn, 200)["results"]
assert List.first(res) == %{
"bounce_rate" => 0,
"date" => "2021-01-01 00:00:00",
"pageviews" => 2,
"visit_duration" => 5,
"visitors" => 1,
"visits" => 1
}
end
test "can filter by event:name", %{conn: conn, site: site} do
populate_stats(site, [
build(:event,

@ -93,7 +93,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -142,7 +142,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 5

@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert plot == [2] ++ List.duplicate(0, 23)
@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -1027,7 +1027,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => true
} = json_response(conn, 200)
@ -1084,7 +1084,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)
@ -1114,7 +1114,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => this_week_plot,
"comparison_plot" => last_week_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)

@ -25,6 +25,37 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "returns top pages by visitors by hostname", %{conn: conn1, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/", hostname: "a.example.com"),
build(:pageview, pathname: "/", hostname: "b.example.com"),
build(:pageview, pathname: "/", hostname: "d.example.com"),
build(:pageview, pathname: "/landing", hostname: "x.example.com", user_id: 123),
build(:pageview, pathname: "/register", hostname: "d.example.com", user_id: 123),
build(:pageview, pathname: "/register", hostname: "d.example.com", user_id: 123),
build(:pageview, pathname: "/register", hostname: "d.example.com"),
build(:pageview, pathname: "/contact", hostname: "e.example.com")
])
filters = Jason.encode!(%{"hostname" => "*.example.com"})
conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
assert json_response(conn, 200) == [
%{"visitors" => 3, "name" => "/"},
%{"visitors" => 2, "name" => "/register"},
%{"visitors" => 1, "name" => "/contact"},
%{"visitors" => 1, "name" => "/landing"}
]
filters = Jason.encode!(%{"hostname" => "d.example.com"})
conn = get(conn1, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
assert json_response(conn, 200) == [
%{"visitors" => 2, "name" => "/register"},
%{"visitors" => 1, "name" => "/"}
]
end
test "returns top pages with :is filter on custom pageview props", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
@ -888,6 +919,132 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "filtering by hostname, excludes a page on different hostname", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
timestamp: ~N[2021-01-01 05:01:00],
pathname: "/about",
hostname: "blog.example.com",
user_id: @user_id
),
build(:pageview,
timestamp: ~N[2021-01-01 05:01:02],
pathname: "/hello",
hostname: "example.com",
user_id: @user_id
),
build(:pageview,
timestamp: ~N[2021-01-01 05:01:02],
pathname: "/about",
hostname: "blog.example.com"
)
])
filters = Jason.encode!(%{"hostname" => "blog.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}"
)
assert json_response(conn, 200) == [
%{
"bounce_rate" => 50,
"name" => "/about",
"pageviews" => 2,
"time_on_page" => nil,
"visitors" => 2
}
]
end
test "calculates bounce rate and time on page for pages when filtered by hostname", %{
conn: conn,
site: site
} do
populate_stats(site, [
# session 1
build(:pageview,
pathname: "/about-blog",
hostname: "blog.example.com",
user_id: @user_id + 1,
timestamp: ~N[2021-01-01 00:01:00]
),
# session 2
build(:pageview,
pathname: "/about-blog",
hostname: "blog.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:01:00]
),
build(:pageview,
pathname: "/about",
hostname: "example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:10:00]
),
build(:pageview,
pathname: "/about-blog",
hostname: "blog.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:15:00]
),
build(:pageview,
pathname: "/exit-blog",
hostname: "blog.example.com",
timestamp: ~N[2021-01-01 00:20:00],
user_id: @user_id
),
build(:pageview,
pathname: "/about",
hostname: "example.com",
timestamp: ~N[2021-01-01 00:22:00],
user_id: @user_id
),
build(:pageview,
pathname: "/exit",
hostname: "example.com",
timestamp: ~N[2021-01-01 00:25:00],
user_id: @user_id
),
# session 3
build(:pageview,
pathname: "/about",
hostname: "example.com",
user_id: @user_id + 2,
timestamp: ~N[2021-01-01 00:01:00]
)
])
filters = Jason.encode!(%{"hostname" => "blog.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&filters=#{filters}"
)
assert json_response(conn, 200) == [
%{
"bounce_rate" => 50,
"name" => "/about-blog",
"pageviews" => 3,
"time_on_page" => 1140.0,
"visitors" => 2
},
%{
"bounce_rate" => nil,
"name" => "/exit-blog",
"pageviews" => 1,
"time_on_page" => nil,
"visitors" => 1
}
]
end
test "doesn't calculate time on page with only single page visits", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),
@ -1227,6 +1384,59 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "returns top entry pages by visitors filtered by hostname",
%{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/page1",
hostname: "en.example.com",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page1",
hostname: "es.example.com",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page2",
hostname: "en.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page2",
hostname: "es.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:15:00]
),
build(:pageview,
pathname: "/exit",
hostname: "es.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:16:00]
),
build(:pageview,
pathname: "/page2",
hostname: "es.example.com",
timestamp: ~N[2021-01-01 23:15:00]
)
])
filters = Jason.encode!(%{"hostname" => "es.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&filters=#{filters}"
)
# We're going to only join sessions where the exit hostname matches the filter
assert json_response(conn, 200) == [
%{"name" => "/page1", "visit_duration" => 0, "visitors" => 1, "visits" => 1},
%{"name" => "/page2", "visit_duration" => 0, "visitors" => 1, "visits" => 1}
]
end
test "bugfix: pagination on /pages filtered by goal", %{conn: conn, site: site} do
populate_stats(
site,
@ -1377,6 +1587,52 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
test "returns top exit pages by visitors filtered by hostname",
%{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,
pathname: "/page1",
hostname: "en.example.com",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page1",
hostname: "es.example.com",
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page1",
hostname: "en.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/page2",
hostname: "es.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:15:00]
),
build(:pageview,
pathname: "/exit",
hostname: "en.example.com",
user_id: @user_id,
timestamp: ~N[2021-01-01 00:16:00]
)
])
filters = Jason.encode!(%{hostname: "es.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&filters=#{filters}"
)
# We're going to only join sessions where the entry hostname matches the filter
assert json_response(conn, 200) ==
[%{"name" => "/page1", "visitors" => 1, "visits" => 1}]
end
test "returns top exit pages filtered by custom pageview props", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,

@ -1221,6 +1221,79 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
]
end
test "returns no top referrers for a custom goal and filtered by hostname",
%{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview,
hostname: "blog.example.com",
referrer_source: "Facebook",
user_id: @user_id
),
build(:pageview,
hostname: "app.example.com",
pathname: "/register",
user_id: @user_id
),
build(:event,
name: "Signup",
hostname: "app.example.com",
pathname: "/register",
user_id: @user_id
)
])
filters = Jason.encode!(%{goal: "Signup", hostname: "app.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
)
assert json_response(conn, 200) == []
end
test "returns top referrers for a custom goal and filtered by hostname (2)",
%{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview,
hostname: "app.example.com",
referrer_source: "Facebook",
pathname: "/register",
user_id: @user_id
),
build(:event,
name: "Signup",
hostname: "app.example.com",
pathname: "/register",
user_id: @user_id
)
])
filters = Jason.encode!(%{goal: "Signup", hostname: "app.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/sources?period=day&filters=#{filters}"
)
assert json_response(conn, 200) == [
%{
"conversion_rate" => 100.0,
"name" => "Facebook",
"total_visitors" => 1,
"visitors" => 1
}
]
end
test "returns top referrers with goal filter + :is prop filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,

@ -236,6 +236,46 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
assert json_response(conn, 200) == []
end
test "returns suggestions for hostnames", %{conn: conn1, user: user} do
{:ok, [site: site]} = create_new_site(%{user: user})
populate_stats(site, [
build(:pageview,
pathname: "/",
hostname: "host-alice.example.com"
),
build(:pageview,
pathname: "/some-other-page",
hostname: "host-bob.example.com",
user_id: 123
),
build(:pageview, pathname: "/exit", hostname: "host-carol.example.com", user_id: 123)
])
conn =
get(
conn1,
"/api/stats/#{site.domain}/suggestions/hostname?q=alice"
)
assert json_response(conn, 200) == [
%{"value" => "host-alice.example.com", "label" => "host-alice.example.com"}
]
conn =
get(
conn1,
"/api/stats/#{site.domain}/suggestions/hostname?q=host"
)
assert json_response(conn, 200) ==
[
%{"label" => "host-alice.example.com", "value" => "host-alice.example.com"},
%{"label" => "host-carol.example.com", "value" => "host-carol.example.com"},
%{"label" => "host-bob.example.com", "value" => "host-bob.example.com"}
]
end
test "returns suggestions for referrers", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview,

@ -6,6 +6,36 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
describe "GET /api/stats/top-stats - default" do
setup [:create_user, :log_in, :create_new_site]
test "returns graph_metric key for graphable top stats", %{conn: conn, site: site} do
[visitors, visits, pageviews, views_per_visit, bounce_rate, visit_duration] =
conn
|> get("/api/stats/#{site.domain}/top-stats")
|> json_response(200)
|> Map.get("top_stats")
assert %{"graph_metric" => "visitors"} = visitors
assert %{"graph_metric" => "visits"} = visits
assert %{"graph_metric" => "pageviews"} = pageviews
assert %{"graph_metric" => "views_per_visit"} = views_per_visit
assert %{"graph_metric" => "bounce_rate"} = bounce_rate
assert %{"graph_metric" => "visit_duration"} = visit_duration
end
test "returns graph_metric key for graphable top stats in realtime mode", %{
conn: conn,
site: site
} do
[current_visitors, unique_visitors, pageviews] =
conn
|> get("/api/stats/#{site.domain}/top-stats?period=realtime")
|> json_response(200)
|> Map.get("top_stats")
refute Map.has_key?(current_visitors, "graph_metric")
assert %{"graph_metric" => "visitors"} = unique_visitors
assert %{"graph_metric" => "pageviews"} = pageviews
end
test "counts distinct user ids", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, user_id: @user_id),
@ -19,7 +49,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{
"name" => "Unique visitors",
"value" => 2
"value" => 2,
"graph_metric" => "visitors"
} in res["top_stats"]
end
@ -36,7 +67,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{
"name" => "Total pageviews",
"value" => 3
"value" => 3,
"graph_metric" => "pageviews"
} in res["top_stats"]
end
@ -59,12 +91,16 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert res["top_stats"] == [
%{"name" => "Unique visitors", "value" => 2},
%{"name" => "Total visits", "value" => 2},
%{"name" => "Total pageviews", "value" => 2},
%{"name" => "Views per visit", "value" => 2.0},
%{"name" => "Bounce rate", "value" => 0},
%{"name" => "Visit duration", "value" => 120}
%{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"},
%{"name" => "Total visits", "value" => 2, "graph_metric" => "visits"},
%{"name" => "Total pageviews", "value" => 2, "graph_metric" => "pageviews"},
%{
"name" => "Views per visit",
"value" => 2.0,
"graph_metric" => "views_per_visit"
},
%{"name" => "Bounce rate", "value" => 0, "graph_metric" => "bounce_rate"},
%{"name" => "Visit duration", "value" => 120, "graph_metric" => "visit_duration"}
]
end
@ -80,7 +116,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Views per visit", "value" => 1.33} in res["top_stats"]
assert %{"name" => "Views per visit", "value" => 1.33, "graph_metric" => "views_per_visit"} in res[
"top_stats"
]
end
test "calculates bounce rate", %{conn: conn, site: site} do
@ -94,7 +132,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Bounce rate", "value" => 50} in res["top_stats"]
assert %{"name" => "Bounce rate", "value" => 50, "graph_metric" => "bounce_rate"} in res[
"top_stats"
]
end
test "calculates average visit duration", %{conn: conn, site: site} do
@ -116,7 +156,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Visit duration", "value" => 450} in res["top_stats"]
assert %{"name" => "Visit duration", "value" => 450, "graph_metric" => "visit_duration"} in res[
"top_stats"
]
end
test "calculates time on page instead when filtered for page", %{conn: conn, site: site} do
@ -339,7 +381,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
filters = Jason.encode!(%{page: "/"})
path = "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"
assert %{"name" => "Bounce rate", "value" => 0} ==
assert %{"name" => "Bounce rate", "value" => 0, "graph_metric" => "bounce_rate"} ==
conn
|> get(path)
|> json_response(200)
@ -435,9 +477,17 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Bounce rate", "value" => 0} in res["top_stats"]
assert %{"name" => "Views per visit", "value" => 0.0} in res["top_stats"]
assert %{"name" => "Visit duration", "value" => 0} in res["top_stats"]
assert %{"name" => "Bounce rate", "value" => 0, "graph_metric" => "bounce_rate"} in res[
"top_stats"
]
assert %{"name" => "Views per visit", "value" => 0.0, "graph_metric" => "views_per_visit"} in res[
"top_stats"
]
assert %{"name" => "Visit duration", "value" => 0, "graph_metric" => "visit_duration"} in res[
"top_stats"
]
end
test "merges imported data into all top stat metrics", %{
@ -468,12 +518,16 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert res["top_stats"] == [
%{"name" => "Unique visitors", "value" => 3},
%{"name" => "Total visits", "value" => 3},
%{"name" => "Total pageviews", "value" => 4},
%{"name" => "Views per visit", "value" => 1.33},
%{"name" => "Bounce rate", "value" => 33},
%{"name" => "Visit duration", "value" => 303}
%{"name" => "Unique visitors", "value" => 3, "graph_metric" => "visitors"},
%{"name" => "Total visits", "value" => 3, "graph_metric" => "visits"},
%{"name" => "Total pageviews", "value" => 4, "graph_metric" => "pageviews"},
%{
"name" => "Views per visit",
"value" => 1.33,
"graph_metric" => "views_per_visit"
},
%{"name" => "Bounce rate", "value" => 33, "graph_metric" => "bounce_rate"},
%{"name" => "Visit duration", "value" => 303, "graph_metric" => "visit_duration"}
]
end
end
@ -504,7 +558,12 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
conn = get(conn, "/api/stats/#{site.domain}/top-stats?period=realtime")
res = json_response(conn, 200)
assert %{"name" => "Unique visitors (last 30 min)", "value" => 2} in res["top_stats"]
assert %{
"name" => "Unique visitors (last 30 min)",
"value" => 2,
"graph_metric" => "visitors"
} in res["top_stats"]
end
test "shows pageviews (last 30 minutes)", %{conn: conn, site: site} do
@ -518,7 +577,10 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
conn = get(conn, "/api/stats/#{site.domain}/top-stats?period=realtime")
res = json_response(conn, 200)
assert %{"name" => "Pageviews (last 30 min)", "value" => 3} in res["top_stats"]
assert %{"name" => "Pageviews (last 30 min)", "value" => 3, "graph_metric" => "pageviews"} in res[
"top_stats"
]
end
test "shows current visitors (last 5 min) with goal filter", %{conn: conn, site: site} do
@ -554,14 +616,77 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
conn = get(conn, "/api/stats/#{site.domain}/top-stats?period=realtime&filters=#{filters}")
res = json_response(conn, 200)
assert %{"name" => "Unique conversions (last 30 min)", "value" => 2} in res["top_stats"]
assert %{"name" => "Total conversions (last 30 min)", "value" => 4} in res["top_stats"]
assert %{
"name" => "Unique conversions (last 30 min)",
"value" => 2,
"graph_metric" => "visitors"
} in res["top_stats"]
assert %{
"name" => "Total conversions (last 30 min)",
"value" => 4,
"graph_metric" => "events"
} in res["top_stats"]
end
end
describe "GET /api/stats/top-stats - filters" do
setup [:create_user, :log_in, :create_new_site]
test "returns graph_metric key for graphable top stats with a page filter", %{
conn: conn,
site: site
} do
filters = Jason.encode!(%{page: "/A"})
[visitors, visits, pageviews, bounce_rate, time_on_page] =
conn
|> get("/api/stats/#{site.domain}/top-stats?filters=#{filters}")
|> json_response(200)
|> Map.get("top_stats")
assert %{"graph_metric" => "visitors"} = visitors
assert %{"graph_metric" => "visits"} = visits
assert %{"graph_metric" => "pageviews"} = pageviews
assert %{"graph_metric" => "bounce_rate"} = bounce_rate
refute Map.has_key?(time_on_page, "graph_metric")
end
test "returns graph_metric key for graphable top stats with a goal filter", %{
conn: conn,
site: site
} do
filters = Jason.encode!(%{goal: "Signup"})
[unique_visitors, unique_conversions, total_conversions, cr] =
conn
|> get("/api/stats/#{site.domain}/top-stats?filters=#{filters}")
|> json_response(200)
|> Map.get("top_stats")
refute Map.has_key?(unique_visitors, "graph_metric")
assert %{"graph_metric" => "visitors"} = unique_conversions
assert %{"graph_metric" => "events"} = total_conversions
assert %{"graph_metric" => "conversion_rate"} = cr
end
test "returns graph_metric key for graphable top stats with a goal filter in realtime mode",
%{conn: conn, site: site} do
filters = Jason.encode!(%{goal: "Signup"})
[current_visitors, unique_conversions, total_conversions] =
conn
|> get("/api/stats/#{site.domain}/top-stats?period=realtime&filters=#{filters}")
|> json_response(200)
|> Map.get("top_stats")
refute Map.has_key?(current_visitors, "graph_metric")
assert %{"graph_metric" => "visitors"} = unique_conversions
assert %{"graph_metric" => "events"} = total_conversions
end
test "returns only visitors from a country based on alpha2 code", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, country_code: "US"),
@ -579,7 +704,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "page glob filter", %{conn: conn, site: site} do
@ -599,7 +726,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "contains (~) filter", %{conn: conn, site: site} do
@ -619,7 +748,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "returns only visitors with specific screen size", %{conn: conn, site: site} do
@ -639,7 +770,46 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "returns only visitors with specific screen size for a given hostname", %{
conn: conn,
site: site
} do
populate_stats(site, [
build(:pageview, screen_size: "Desktop", hostname: "blog.example.com"),
build(:pageview, screen_size: "Desktop", hostname: "example.com", user_id: @user_id),
build(:pageview, screen_size: "Desktop", hostname: "blog.example.com", user_id: @user_id),
build(:pageview,
screen_size: "Desktop",
hostname: "blog.example.com",
user_id: @user_id + 1
),
build(:pageview, screen_size: "Desktop", hostname: "example.com", user_id: @user_id + 1),
build(:pageview, screen_size: "Mobile", hostname: "blog.example.com")
])
filters = Jason.encode!(%{screen: "Desktop", hostname: "blog.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/top-stats?period=month&filters=#{filters}"
)
res =
json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 3, "graph_metric" => "visitors"} in res[
"top_stats"
]
assert %{"name" => "Total visits", "value" => 3, "graph_metric" => "visits"} in res[
"top_stats"
]
end
test "returns only visitors with specific browser", %{conn: conn, site: site} do
@ -659,7 +829,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "returns only visitors with specific operating system", %{conn: conn, site: site} do
@ -679,7 +851,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2} in res["top_stats"]
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "returns number of visits from one specific referral source", %{conn: conn, site: site} do
@ -712,7 +886,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Total visits", "value" => 2} in res["top_stats"]
assert %{"name" => "Total visits", "value" => 2, "graph_metric" => "visits"} in res[
"top_stats"
]
end
test "does not return the views_per_visit metric when a page filter is applied", %{
@ -738,6 +914,96 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
_ -> false
end)
end
test "hostname exact filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/index", hostname: "example.com"),
build(:pageview, pathname: "/index", hostname: "example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post1", hostname: "blog.example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post2", hostname: "blog.example.com")
])
filters = Jason.encode!(%{hostname: "example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/top-stats?period=month&filters=#{filters}"
)
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"} in res[
"top_stats"
]
assert %{"name" => "Total pageviews", "value" => 2, "graph_metric" => "pageviews"} in res[
"top_stats"
]
end
test "hostname glob filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/index", hostname: "example.com"),
build(:pageview, pathname: "/index", hostname: "example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post1", hostname: "blog.example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post2", hostname: "blog.example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post2", hostname: "blog.example.com"),
build(:pageview, pathname: "/blog/post2", hostname: "about.example.com")
])
filters = Jason.encode!(%{hostname: "*example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/top-stats?period=month&filters=#{filters}"
)
res =
json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 4, "graph_metric" => "visitors"} in res[
"top_stats"
]
assert %{"name" => "Total pageviews", "value" => 6, "graph_metric" => "pageviews"} in res[
"top_stats"
]
end
test "hostname glob subdomain filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/index", hostname: "example.com"),
build(:pageview, pathname: "/index", hostname: "example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post1", hostname: "blog.example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post2", hostname: "blog.example.com", user_id: @user_id),
build(:pageview, pathname: "/blog/post3", hostname: "blog.example.com"),
build(:pageview,
pathname: "/blog/post2",
hostname: "blog.example.com",
user_id: 100_002_378_237
)
])
filters = Jason.encode!(%{hostname: "*.example.com"})
conn =
get(
conn,
"/api/stats/#{site.domain}/top-stats?period=month&filters=#{filters}"
)
res = json_response(conn, 200)
assert %{"name" => "Unique visitors", "value" => 3, "graph_metric" => "visitors"} in res[
"top_stats"
]
assert %{"name" => "Total pageviews", "value" => 4, "graph_metric" => "pageviews"} in res[
"top_stats"
]
end
end
describe "GET /api/stats/top-stats - filtered for goal" do
@ -782,7 +1048,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Unique conversions", "value" => 1} in res["top_stats"]
assert %{"name" => "Unique conversions", "value" => 1, "graph_metric" => "visitors"} in res[
"top_stats"
]
end
test "returns conversion rate", %{conn: conn, site: site} do
@ -803,7 +1071,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Conversion rate", "value" => 33.3} in res["top_stats"]
assert %{"name" => "Conversion rate", "value" => 33.3, "graph_metric" => "conversion_rate"} in res[
"top_stats"
]
end
test "returns conversion rate without change when comparison mode disallowed", %{
@ -827,7 +1097,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Conversion rate", "value" => 33.3} in res["top_stats"]
assert %{"name" => "Conversion rate", "value" => 33.3, "graph_metric" => "conversion_rate"} in res[
"top_stats"
]
end
@tag :full_build_only
@ -864,12 +1136,14 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{
"name" => "Average revenue",
"value" => %{"long" => "$1,659.50", "short" => "$1.7K"}
"value" => %{"long" => "$1,659.50", "short" => "$1.7K"},
"graph_metric" => "average_revenue"
} in top_stats
assert %{
"name" => "Total revenue",
"value" => %{"long" => "$3,319.00", "short" => "$3.3K"}
"value" => %{"long" => "$3,319.00", "short" => "$3.3K"},
"graph_metric" => "total_revenue"
} in top_stats
end
@ -921,12 +1195,14 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{
"name" => "Average revenue",
"value" => %{"long" => "$1,659.50", "short" => "$1.7K"}
"value" => %{"long" => "$1,659.50", "short" => "$1.7K"},
"graph_metric" => "average_revenue"
} in top_stats
assert %{
"name" => "Total revenue",
"value" => %{"long" => "$6,638.00", "short" => "$6.6K"}
"value" => %{"long" => "$6,638.00", "short" => "$6.6K"},
"graph_metric" => "total_revenue"
} in top_stats
end
@ -993,7 +1269,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
describe "GET /api/stats/top-stats - with comparisons" do
setup [:create_user, :log_in, :create_new_site, :add_imported_data]
test "defaults to previous period when comparison is not set", %{site: site, conn: conn} do
test "does not return comparisons by default", %{site: site, conn: conn} do
populate_stats(site, [
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),
@ -1006,7 +1282,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"name" => "Total visits", "value" => 3} in res["top_stats"]
assert %{"name" => "Total visits", "value" => 3, "graph_metric" => "visits"} in res[
"top_stats"
]
end
test "returns comparison data when mode is custom", %{site: site, conn: conn} do
@ -1028,7 +1306,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
res = json_response(conn, 200)
assert %{"change" => 0, "comparison_value" => 3, "name" => "Total visits", "value" => 3} in res[
assert %{
"change" => 0,
"comparison_value" => 3,
"name" => "Total visits",
"value" => 3,
"graph_metric" => "visits"
} in res[
"top_stats"
]
end
@ -1087,7 +1371,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{"top_stats" => top_stats, "with_imported" => true} = json_response(conn, 200)
assert %{"change" => 100, "comparison_value" => 2, "name" => "Total visits", "value" => 4} in top_stats
assert %{
"change" => 100,
"comparison_value" => 2,
"name" => "Total visits",
"value" => 4,
"graph_metric" => "visits"
} in top_stats
end
test "does not compare imported data when with_imported is set to false", %{
@ -1122,7 +1412,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
assert %{"top_stats" => top_stats, "with_imported" => false} = json_response(conn, 200)
assert %{"change" => 100, "comparison_value" => 0, "name" => "Total visits", "value" => 4} in top_stats
assert %{
"change" => 100,
"comparison_value" => 0,
"name" => "Total visits",
"value" => 4,
"graph_metric" => "visits"
} in top_stats
end
test "compares conversion rates", %{conn: conn, site: site} do
@ -1150,7 +1446,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
"change" => -33.4,
"comparison_value" => 66.7,
"name" => "Conversion rate",
"value" => 33.3
"value" => 33.3,
"graph_metric" => "conversion_rate"
} in res["top_stats"]
end
end

@ -5,6 +5,7 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
import Mox
import Plausible.Test.Support.HTML
alias Plausible.HTTPClient
alias Plausible.Imported.SiteImport
require Plausible.Imported.SiteImport
@ -22,6 +23,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"start_date" => "2020-02-22",
"end_date" => "2022-09-22",
"legacy" => "true"
})
|> html_response(200)
@ -32,6 +35,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
access_token: "token",
refresh_token: "foo",
expires_at: "2022-09-22T20:01:37.112777",
start_date: "2020-02-22",
end_date: "2022-09-22",
legacy: "true"
)
|> String.replace("&", "&amp;")
@ -100,21 +105,368 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
assert response =~ "GA4 - Flood-It! (properties/153293282)"
assert response =~ "GA4 - Google Merch Shop (properties/213025502)"
end
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} on auth error with flash error (legacy: #{legacy})", %{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
{:error, %HTTPClient.Non200Error{reason: %{status: 403, body: %{}}}}
end
)
conn =
conn
|> get("/#{site.domain}/import/google-analytics/property-or-view", %{
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"We were unable to authenticate your Google Analytics account"
end
test "redirects to #{view} on list retrival failure with flash error (legacy: #{legacy})",
%{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
{:error,
%HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}}
end
)
conn =
conn
|> get("/#{site.domain}/import/google-analytics/property-or-view", %{
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"We were unable to list your Google Analytics properties and views"
end
end
end
describe "GET /:website/import/google-analytics/confirm" do
describe "POST /:website/import/google-analytics/property-or-view" do
setup [:create_user, :log_in, :create_new_site]
test "renders confirmation form for Universal Analytics import", %{conn: conn, site: site} do
for legacy <- ["true", "false"] do
test "redirects to user metrics notice (UA legacy: #{legacy})", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga_start_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga_end_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
conn =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "57238190",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) =~
"/#{URI.encode_www_form(site.domain)}/import/google-analytics/user-metric"
end
test "redirects to confirmation (UA legacy: #{legacy})", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga_start_date_later.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga_end_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
conn =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "57238190",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) =~
"/#{URI.encode_www_form(site.domain)}/import/google-analytics/confirm"
end
end
test "redirects to confirmation (GA4)", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga4_end_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
conn =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => "false"
})
assert redirected_to(conn, 302) =~
"/#{URI.encode_www_form(site.domain)}/import/google-analytics/confirm"
end
test "renders error when no time window to import available", %{conn: conn, site: site} do
start_date = ~D[2022-01-12]
end_date = ~D[2024-03-13]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _headers, _params ->
body = "fixture/ga_start_date.json" |> File.read!() |> Jason.decode!()
fn _url, _opts, _params ->
body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
body = "fixture/ga4_end_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
body = "fixture/ga4_list_properties.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
body = "fixture/ga_list_views.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
response =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => "false"
})
|> html_response(200)
assert response =~
"Imported data time range is completely overlapping with existing data. Nothing to import."
end
test "renders error when there's no data to import", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
{:ok, %Finch.Response{body: %{"reports" => []}, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
{:ok, %Finch.Response{body: %{"reports" => []}, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
body = "fixture/ga4_list_properties.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _opts ->
body = "fixture/ga_list_views.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
response =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => "false"
})
|> html_response(200)
assert response =~ "No data found. Nothing to import."
end
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} on failed property/view choice with flash error (legacy: #{legacy})",
%{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
{:error,
%HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}}
end
)
conn =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"We were unable to retrieve information from Google Analytics"
end
test "redirects to #{view} on expired authentication with flash error (legacy: #{legacy})",
%{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _opts, _params ->
{:error, %HTTPClient.Non200Error{reason: %{status: 403, body: "Access denied"}}}
end
)
conn =
conn
|> post("/#{site.domain}/import/google-analytics/property-or-view", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"Google Analytics authentication seems to have expired."
end
end
end
describe "GET /:website/import/google-analytics/confirm" do
setup [:create_user, :log_in, :create_new_site]
test "renders confirmation form for Universal Analytics import", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:get,
@ -131,6 +483,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"start_date" => "2012-01-18",
"end_date" => "2022-09-22",
"legacy" => "true"
})
|> html_response(200)
@ -151,20 +505,10 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
assert text_of_attr(response, ~s|input[name=start_date]|, "value") == "2012-01-18"
assert text_of_attr(response, ~s|input[name=end_date]|, "value") ==
Date.to_iso8601(Date.utc_today())
assert text_of_attr(response, ~s|input[name=end_date]|, "value") == "2022-09-22"
end
test "renders confirmation form for Google Analytics 4 import", %{conn: conn, site: site} do
expect(
Plausible.HTTPClient.Mock,
:post,
fn _url, _headers, _params ->
body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!()
{:ok, %Finch.Response{body: body, status: 200}}
end
)
expect(
Plausible.HTTPClient.Mock,
:get,
@ -181,6 +525,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"start_date" => "2024-02-22",
"end_date" => "2024-02-26",
"legacy" => "true"
})
|> html_response(200)
@ -202,8 +548,82 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
assert text_of_attr(response, ~s|input[name=start_date]|, "value") == "2024-02-22"
assert text_of_attr(response, ~s|input[name=end_date]|, "value") ==
Date.to_iso8601(Date.utc_today())
assert text_of_attr(response, ~s|input[name=end_date]|, "value") == "2024-02-26"
end
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} on failed property/view retrieval with flash error (legacy: #{legacy})",
%{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _params ->
{:error,
%HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}}
end
)
conn =
conn
|> get("/#{site.domain}/import/google-analytics/confirm", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"start_date" => "2024-02-22",
"end_date" => "2024-02-26",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"We were unable to retrieve information from Google Analytics"
end
test "redirects to #{view} on expired authentication with flash error (legacy: #{legacy})",
%{
conn: conn,
site: site
} do
expect(
Plausible.HTTPClient.Mock,
:get,
fn _url, _params ->
{:error, %HTTPClient.Non200Error{reason: %{status: 403, body: "Access denied"}}}
end
)
conn =
conn
|> get("/#{site.domain}/import/google-analytics/confirm", %{
"property_or_view" => "properties/428685906",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"start_date" => "2024-02-22",
"end_date" => "2024-02-26",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"Google Analytics authentication seems to have expired."
end
end
end
@ -332,5 +752,44 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
}
)
end
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{
conn: conn,
site: site
} do
start_date = ~D[2022-01-12]
end_date = ~D[2024-03-13]
_existing_import =
insert(:site_import,
site: site,
start_date: start_date,
end_date: end_date,
status: :completed
)
conn =
post(conn, "/#{site.domain}/settings/google-import", %{
"property_or_view" => "123456",
"start_date" => "2023-03-01",
"end_date" => "2022-03-01",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => unquote(legacy)
})
assert redirected_to(conn, 302) ==
PlausibleWeb.Router.Helpers.site_path(
conn,
unquote(view),
site.domain
)
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"Import failed. No data could be imported because date range overlaps with existing data."
end
end
end
end

@ -90,31 +90,6 @@ defmodule Plausible.TestUtils do
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews)
end
def create_events(events) do
events =
Enum.map(events, fn event ->
Factory.build(:event, event)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events)
end
def create_sessions(sessions) do
sessions =
Enum.map(sessions, fn session ->
Factory.build(:ch_session, session)
|> Map.from_struct()
|> Map.delete(:__meta__)
|> update_in([:timestamp], &to_naive_truncate/1)
|> update_in([:start], &to_naive_truncate/1)
end)
Plausible.IngestRepo.insert_all(Plausible.ClickhouseSessionV2, sessions)
end
def log_in(%{user: user, conn: conn}) do
conn =
init_session(conn)
@ -198,7 +173,7 @@ defmodule Plausible.TestUtils do
session = Plausible.Session.CacheStore.on_event(event_params, event_params, nil)
event_params
|> Map.merge(session)
|> Plausible.ClickhouseEventV2.merge_session(session)
|> Plausible.Event.WriteBuffer.insert()
end

@ -1,9 +1,16 @@
{:ok, _} = Application.ensure_all_started(:ex_machina)
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:window_time_on_page)
FunWithFlags.enable(:imports_exports)
FunWithFlags.enable(:shield_pages)
# Temporary flag to test `experimental_reduced_joins` flag on all tests.
if System.get_env("TEST_EXPERIMENTAL_REDUCED_JOINS") == "1" do
FunWithFlags.enable(:experimental_reduced_joins)
else
FunWithFlags.disable(:experimental_reduced_joins)
end
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
if Mix.env() == :small_test do

Loading…
Cancel
Save