Compare commits

..

2 Commits

Author SHA1 Message Date
Adam Rutkowski a88530e351 Migration: Shield page rules 3 years ago
Adam Rutkowski 33b5c10654 Revert "Pages shield (#3918)"
This reverts commit 53f94a9f82.
3 years ago

@ -15,25 +15,14 @@ env:
jobs: jobs:
build: build:
name: "Build and test (${{ matrix.mix_env }}, ${{ matrix.postgres_image }}${{ matrix.test_experimental_reduced_joins == '1' && ', experimental_reduced_joins' || '' }})" name: Build and test
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
mix_env: ['test', 'small_test'] mix_env: ['test', 'small_test']
postgres_image: ['postgres:16'] postgres_image: ['postgres:15', 'postgres:16']
test_experimental_reduced_joins: ['0']
include:
- mix_env: 'test'
postgres_image: 'postgres:15'
test_experimental_reduced_joins: '0'
- mix_env: 'test'
postgres_image: 'postgres:16'
test_experimental_reduced_joins: '1'
env: env:
MIX_ENV: ${{ matrix.mix_env }} MIX_ENV: ${{ matrix.mix_env }}
TEST_EXPERIMENTAL_REDUCED_JOINS: ${{ matrix.test_experimental_reduced_joins }}
services: services:
postgres: postgres:
image: ${{ matrix.postgres_image }} image: ${{ matrix.postgres_image }}

@ -1,28 +0,0 @@
name: Validate migrations
on:
pull_request:
paths:
- 'priv/repo/migrations/**'
- 'priv/ingest_repo/migrations/**'
jobs:
validate:
name: App code does not change at the same time
runs-on: ubuntu-latest
steps:
- uses: dorny/paths-filter@v3
id: changes
with:
list-files: json
filters: |
lib:
- 'lib/**'
- 'extra/**'
- 'config/**'
- if: steps.changes.outputs.lib == 'true'
run: |
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
exit 1

@ -2,8 +2,6 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
### Added ### Added
- Pages Block List in Site Settings
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export - Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export
- Ability to display total conversions (with a goal filter) on the main graph - Ability to display total conversions (with a goal filter) on the main graph
- Add `conversion_rate` to Stats API Timeseries and on the main graph - Add `conversion_rate` to Stats API Timeseries and on the main graph
@ -57,8 +55,6 @@ All notable changes to this project will be documented in this file.
- Validate metric isn't queried multiple times - Validate metric isn't queried multiple times
### Fixed ### Fixed
- Creating many sites no longer leads to cookie overflow
- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns
- Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown - Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown
- Fix `conversion_rate` metric in a `browser_versions` breakdown - Fix `conversion_rate` metric in a `browser_versions` breakdown
- Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division) - Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division)

@ -22,7 +22,6 @@ if (container) {
conversionsOptedOut: container.dataset.conversionsOptedOut === 'true', conversionsOptedOut: container.dataset.conversionsOptedOut === 'true',
funnelsOptedOut: container.dataset.funnelsOptedOut === 'true', funnelsOptedOut: container.dataset.funnelsOptedOut === 'true',
propsOptedOut: container.dataset.propsOptedOut === 'true', propsOptedOut: container.dataset.propsOptedOut === 'true',
revenueGoals: JSON.parse(container.dataset.revenueGoals),
funnels: JSON.parse(container.dataset.funnels), funnels: JSON.parse(container.dataset.funnels),
statsBegin: container.dataset.statsBegin, statsBegin: container.dataset.statsBegin,
nativeStatsBegin: container.dataset.nativeStatsBegin, nativeStatsBegin: container.dataset.nativeStatsBegin,

@ -59,7 +59,6 @@ export function parseQuery(querystring, site) {
'region': q.get('region'), 'region': q.get('region'),
'city': q.get('city'), 'city': q.get('city'),
'page': q.get('page'), 'page': q.get('page'),
'hostname': q.get('hostname'),
'entry_page': q.get('entry_page'), 'entry_page': q.get('entry_page'),
'exit_page': q.get('exit_page') 'exit_page': q.get('exit_page')
} }

@ -1,41 +1,19 @@
import numberFormatter, {durationFormatter} from '../../util/number-formatter' import numberFormatter, {durationFormatter} from '../../util/number-formatter'
import { parsePrefix } from '../../util/filters'
export function getGraphableMetrics(query, site) { export const METRIC_MAPPING = {
const isRealtime = query.period === 'realtime' 'Unique visitors (last 30 min)': 'visitors',
const goalFilter = query.filters.goal 'Pageviews (last 30 min)': 'pageviews',
const pageFilter = query.filters.page 'Unique visitors': 'visitors',
'Visit duration': 'visit_duration',
if (isRealtime && goalFilter) { 'Total pageviews': 'pageviews',
return ["visitors"] 'Views per visit': 'views_per_visit',
} else if (isRealtime) { 'Total visits': 'visits',
return ["visitors", "pageviews"] 'Bounce rate': 'bounce_rate',
} else if (goalFilter && canGraphRevenueMetrics(goalFilter, site)) { 'Unique conversions': 'conversions',
return ["visitors", "events", "average_revenue", "total_revenue", "conversion_rate"] 'Total conversions': 'events',
} else if (goalFilter) { 'Conversion rate': 'conversion_rate',
return ["visitors", "events", "conversion_rate"] 'Average revenue': 'average_revenue',
} else if (pageFilter) { 'Total revenue': 'total_revenue',
return ["visitors", "visits", "pageviews", "bounce_rate", "time_on_page"]
} else {
return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]
}
}
// Revenue metrics can only be graphed if:
// * The query is filtered by at least one revenue goal
// * All revenue goals in filter have the same currency
function canGraphRevenueMetrics(goalFilter, site) {
const goalsInFilter = parsePrefix(goalFilter).values
const revenueGoalsInFilter = site.revenueGoals.filter((rg) => {
return goalsInFilter.includes(rg.event_name)
})
const singleCurrency = revenueGoalsInFilter.every((rg) => {
return rg.currency === revenueGoalsInFilter[0].currency
})
return revenueGoalsInFilter.length > 0 && singleCurrency
} }
export const METRIC_LABELS = { export const METRIC_LABELS = {
@ -66,6 +44,14 @@ export const METRIC_FORMATTER = {
'average_revenue': numberFormatter, 'average_revenue': numberFormatter,
} }
export const LoadingState = {
loading: 'loading',
refreshing: 'refreshing',
loaded: 'loaded',
isLoadingOrRefreshing: function (state) { return [this.loading, this.refreshing].includes(state) },
isLoadedOrRefreshing: function (state) { return [this.loaded, this.refreshing].includes(state) }
}
const buildComparisonDataset = function(comparisonPlot) { const buildComparisonDataset = function(comparisonPlot) {
if (!comparisonPlot) return [] if (!comparisonPlot) return []

@ -6,7 +6,7 @@ import * as storage from '../../util/storage'
import { isKeyPressed } from '../../keybinding.js' import { isKeyPressed } from '../../keybinding.js'
import { monthsBetweenDates } from '../../util/date.js' import { monthsBetweenDates } from '../../util/date.js'
const INTERVAL_LABELS = { export const INTERVAL_LABELS = {
'minute': 'Minutes', 'minute': 'Minutes',
'hour': 'Hours', 'hour': 'Hours',
'date': 'Days', 'date': 'Days',
@ -14,19 +14,11 @@ const INTERVAL_LABELS = {
'month': 'Months' 'month': 'Months'
} }
function validIntervals(site, query) { export const getStoredInterval = function(period, domain) {
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
return ["week", "month"]
} else {
return site.validIntervalsByPeriod[query.period]
}
}
function getStoredInterval(period, domain) {
return storage.getItem(`interval__${period}__${domain}`) return storage.getItem(`interval__${period}__${domain}`)
} }
function storeInterval(period, domain, interval) { export const storeInterval = function(period, domain, interval) {
storage.setItem(`interval__${period}__${domain}`, interval) storage.setItem(`interval__${period}__${domain}`, interval)
} }
@ -41,47 +33,37 @@ function subscribeKeybinding(element) {
}, [handleKeyPress]) }, [handleKeyPress])
} }
export const getCurrentInterval = function(site, query) { function DropdownItem({ option, currentInterval, updateInterval }) {
const options = validIntervals(site, query) return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
const storedInterval = getStoredInterval(query.period, site.domain) {({ active }) => (
const defaultInterval = [...options].pop() <span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
if (storedInterval && options.includes(storedInterval)) { 'text-gray-700 dark:text-gray-200': !active,
return storedInterval 'font-bold cursor-none select-none': option == currentInterval,
} else { }, 'block px-4 py-2 text-sm')}>
return defaultInterval {INTERVAL_LABELS[option]}
} </span>
)}
</Menu.Item>
)
} }
export function IntervalPicker({ query, site, onIntervalUpdate }) { export function IntervalPicker({ graphData, query, site, updateInterval }) {
if (query.period == 'realtime') return null if (query.period == 'realtime') return null
const menuElement = React.useRef(null) const menuElement = React.useRef(null)
const options = validIntervals(site, query)
const currentInterval = getCurrentInterval(site, query)
subscribeKeybinding(menuElement) subscribeKeybinding(menuElement)
function updateInterval(interval) { let currentInterval = graphData?.interval
storeInterval(query.period, site.domain, interval)
onIntervalUpdate(interval) let options = site.validIntervalsByPeriod[query.period]
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
options = ["week", "month"]
} }
function renderDropdownItem(option) { if (!options.includes(currentInterval)) {
return ( currentInterval = [...options].pop()
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
} }
return ( return (
@ -103,7 +85,7 @@ export function IntervalPicker({ query, site, onIntervalUpdate }) {
leaveFrom="opacity-100 scale-100" leaveFrom="opacity-100 scale-100"
leaveTo="opacity-0 scale-95"> leaveTo="opacity-0 scale-95">
<Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static> <Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static>
{options.map(renderDropdownItem)} {options.map((option) => DropdownItem({ option, currentInterval, updateInterval }))}
</Menu.Items> </Menu.Items>
</Transition> </Transition>
</> </>

@ -1,247 +0,0 @@
import React from 'react';
import { withRouter } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
import dateFormatter from './date-formatter';
import FadeIn from '../../fade-in';
import classNames from 'classnames';
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
}
getGraphMetric() {
let metric = this.props.graphData.metric
if (metric == 'visitors' && this.props.query.filters.goal) {
return 'conversions'
} else {
return metric
}
}
regenerateChart() {
const { graphData, query } = this.props
const metric = this.getGraphMetric()
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.maybeHopToHoveredPeriod.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
window.addEventListener('mousemove', this.repositionTooltip);
}
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
}
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
maybeHopToHoveredPeriod(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
render() {
const { graphData } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
)
}
}
export default withRouter(LineGraph)

@ -1,17 +0,0 @@
import React from "react"
export default function SamplingNotice({topStatData}) {
const samplePercent = topStatData?.samplePercent
if (samplePercent && samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
} else {
return null
}
}

@ -1,53 +0,0 @@
import React, { useState } from "react"
import * as api from '../../api'
import { getCurrentInterval } from "./interval-picker"
export default function StatsExport({site, query}) {
const [exporting, setExporting] = useState(false)
function startExport() {
setExporting(true)
document.cookie = "exporting="
pollExportReady()
}
function pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(pollExportReady, 1000)
} else {
setExporting(false)
}
}
function renderLoading() {
return (
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
)
}
function renderExportLink() {
const interval = getCurrentInterval(site, query)
const queryParams = api.serializeQuery(query, [{ interval }])
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
return (
<a href={endpoint} download onClick={startExport}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
<polyline points="7 10 12 15 17 10"></polyline>
<line x1="12" y1="15" x2="12" y2="3"></line>
</svg>
</a>
)
}
return (
<div className="w-4 h-4 mx-2">
{exporting && renderLoading()}
{!exporting && renderExportLink()}
</div>
)
}

@ -3,9 +3,8 @@ import { Tooltip } from '../../util/tooltip'
import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load' import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load'
import classNames from "classnames"; import classNames from "classnames";
import numberFormatter, { durationFormatter } from '../../util/number-formatter' import numberFormatter, { durationFormatter } from '../../util/number-formatter'
import * as storage from '../../util/storage' import { METRIC_MAPPING } from './graph-util'
import { formatDateRange } from '../../util/date.js' import { formatDateRange } from '../../util/date.js'
import { getGraphableMetrics } from "./graph-util.js";
function Maybe({condition, children}) { function Maybe({condition, children}) {
if (condition) { if (condition) {
@ -15,103 +14,99 @@ function Maybe({condition, children}) {
} }
} }
function renderPercentageComparison(name, comparison, forceDarkBg = false) { export default class TopStats extends React.Component {
const formattedComparison = numberFormatter(Math.abs(comparison)) renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg, const defaultClassName = classNames({
"pl-2 text-xs text-gray-100": forceDarkBg "pl-2 text-xs dark:text-gray-100": !forceDarkBg,
}) "pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg, const noChangeClassName = classNames({
"pl-2 text-xs text-gray-300": forceDarkBg "pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
}) "pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500' if (comparison > 0) {
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span> const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
} else if (comparison < 0) { return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400' } else if (comparison < 0) {
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span> const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
} else if (comparison === 0) { return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
return <span className={noChangeClassName}>&#12336; 0%</span> } else if (comparison === 0) {
} else { return <span className={noChangeClassName}>&#12336; 0%</span>
return null } else {
return null
}
} }
}
function topStatNumberShort(name, value) { topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) { if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value) return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) { } else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%' return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) { } else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short return value?.short
} else { } else {
return numberFormatter(value) return numberFormatter(value)
}
} }
}
function topStatNumberLong(name, value) { topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) { if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value) return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) { } else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%' return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) { } else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long return value?.long
} else { } else {
return (value || 0).toLocaleString() return (value || 0).toLocaleString()
}
} }
}
export default function TopStats(props) {
const {site, query, data, onMetricUpdate, tooltipBoundary, lastLoadTimestamp} = props
function tooltip(stat) { topStatTooltip(stat, query) {
let statName = stat.name.toLowerCase() let statName = stat.name.toLowerCase()
statName = stat.value === 1 ? statName.slice(0, -1) : statName statName = stat.value === 1 ? statName.slice(0, -1) : statName
return ( return (
<div> <div>
{query.comparison && <div className="whitespace-nowrap"> {query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} vs. {topStatNumberLong(stat.name, stat.comparison_value)} {statName} {this.topStatNumberLong(stat.name, stat.value)} vs. {this.topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{renderPercentageComparison(stat.name, stat.change, true)}</span> <span className="ml-2">{this.renderPercentageComparison(stat.name, stat.change, true)}</span>
</div>} </div>}
{!query.comparison && <div className="whitespace-nowrap"> {!query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} {statName} {this.topStatNumberLong(stat.name, stat.value)} {statName}
</div>} </div>}
{stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp}/>s ago</p>} {stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={this.props.lastLoadTimestamp}/>s ago</p>}
</div> </div>
) )
} }
function canMetricBeGraphed(stat) { canMetricBeGraphed(stat) {
const graphableMetrics = getGraphableMetrics(query, site) const isTotalUniqueVisitors = this.props.query.filters.goal && stat.name === 'Unique visitors'
return stat.graph_metric && graphableMetrics.includes(stat.graph_metric) const isKnownMetric = Object.keys(METRIC_MAPPING).includes(stat.name)
return isKnownMetric && !isTotalUniqueVisitors
} }
function maybeUpdateMetric(stat) { maybeUpdateMetric(stat) {
if (canMetricBeGraphed(stat)) { if (this.canMetricBeGraphed(stat)) {
storage.setItem(`metric__${site.domain}`, stat.graph_metric) this.props.updateMetric(METRIC_MAPPING[stat.name])
onMetricUpdate(stat.graph_metric)
} }
} }
function blinkingDot() { blinkingDot() {
return ( return (
<div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div> <div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div>
) )
} }
function getStoredMetric() { renderStatName(stat) {
return storage.getItem(`metric__${site.domain}`) const { metric } = this.props
} const isSelected = metric === METRIC_MAPPING[stat.name]
function renderStatName(stat) {
const isSelected = stat.graph_metric === getStoredMetric()
const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g) const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g)
@ -128,45 +123,48 @@ export default function TopStats(props) {
) )
} }
function renderStat(stat, index) { render() {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', { const { topStatData, query, site } = this.props
'cursor-pointer': canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0, const stats = topStatData && topStatData.top_stats.map((stat, index) => {
'border-r lg:border-r-0': index % 2 === 0
}) const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': this.canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
return (
<Tooltip key={stat.name} info={this.topStatTooltip(stat, query)} className={className} onClick={() => { this.maybeUpdateMetric(stat) }} boundary={this.props.tooltipBoundary}>
{this.renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={METRIC_MAPPING[stat.name]}>{this.topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ this.renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, topStatData.from, topStatData.to) }</p>
</Maybe>
</div>
return (
<Tooltip key={stat.name} info={tooltip(stat, query)} className={className} onClick={() => { maybeUpdateMetric(stat) }} boundary={tooltipBoundary}>
{renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={stat.graph_metric}>{topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}> <Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, data.from, data.to) }</p> <div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ this.topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, topStatData.comparing_from, topStatData.comparing_to) }</p>
</div>
</Maybe> </Maybe>
</div>
<Maybe condition={query.comparison}>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, data.comparing_from, data.comparing_to) }</p>
</div> </div>
</Maybe> </Tooltip>
</div> )
</Tooltip> })
)
}
const stats = data && data.top_stats.map(renderStat) if (stats && query && query.period === 'realtime') {
stats.push(this.blinkingDot())
}
if (stats && query.period === 'realtime') { return stats || null;
stats.push(blinkingDot())
} }
return stats || null;
} }

@ -1,153 +1,534 @@
import React, { useState, useEffect, useRef, useCallback } from 'react'; import React from 'react';
import { withRouter, Link } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import * as api from '../../api' import * as api from '../../api'
import * as storage from '../../util/storage' import * as storage from '../../util/storage'
import { getGraphableMetrics } from './graph-util' import LazyLoader from '../../components/lazy-loader'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_MAPPING, METRIC_LABELS, METRIC_FORMATTER, LoadingState } from './graph-util'
import dateFormatter from './date-formatter';
import TopStats from './top-stats'; import TopStats from './top-stats';
import { IntervalPicker, getCurrentInterval } from './interval-picker' import { IntervalPicker, getStoredInterval, storeInterval } from './interval-picker';
import StatsExport from './stats-export'
import WithImportedSwitch from './with-imported-switch';
import SamplingNotice from './sampling-notice';
import FadeIn from '../../fade-in'; import FadeIn from '../../fade-in';
import * as url from '../../util/url' import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input' import { isComparisonEnabled } from '../../comparison-input'
import LineGraphWithRouter from './line-graph'
function fetchTopStats(site, query) { const calculateMaximumY = function(dataset) {
const q = { ...query } const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (!isComparisonEnabled(q.comparison)) { if (yAxisValues) {
q.comparison = 'previous_period' return Math.max(...yAxisValues)
} else {
return 1
} }
return api.get(url.apiPath(site, '/top-stats'), q)
} }
function fetchMainGraph(site, query, metric, interval) { class LineGraph extends React.Component {
const params = {metric, interval} constructor(props) {
return api.get(url.apiPath(site, '/main-graph'), query, params) super(props);
} this.boundary = React.createRef()
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
this.state = {
exported: false
};
}
export default function VisitorGraph(props) { regenerateChart() {
const {site, query, lastLoadTimestamp} = props const { graphData, metric, query } = this.props
const isRealtime = query.period === 'realtime' const graphEl = document.getElementById("main-graph-canvas")
const isDarkTheme = document.querySelector('html').classList.contains('dark') || false this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
const topStatsBoundary = useRef(null) return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.onClick.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const [topStatData, setTopStatData] = useState(null) const hasMultipleYears =
const [topStatsLoading, setTopStatsLoading] = useState(true) graphData.labels
const [graphData, setGraphData] = useState(null) .filter((date) => typeof date === 'string')
const [graphLoading, setGraphLoading] = useState(true) .map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
// This state is explicitly meant for the situation where either graph interval if (graphData.interval === 'hour' && query.period !== 'day') {
// or graph metric is changed. That results in behaviour where Top Stats stay const date = dateFormatter({
// intact, but the graph container alone will display a loading spinner for as interval: "date",
// long as new graph data is fetched. longForm: false,
const [graphRefreshing, setGraphRefreshing] = useState(false) period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const onIntervalUpdate = useCallback((newInterval) => { // Returns a combination of date and hour. This is because
setGraphData(null) // small intervals like hour may return multiple days
setGraphRefreshing(true) // depending on the query period.
fetchGraphData(getStoredMetric(), newInterval) return `${date}, ${hour}`
}, [query]) }
const onMetricUpdate = useCallback((newMetric) => { if (graphData.interval === 'minute' && query.period !== 'realtime') {
setGraphData(null) return dateFormatter({
setGraphRefreshing(true) interval: "hour", longForm: false, period: query.period,
fetchGraphData(newMetric, getCurrentInterval(site, query)) })(this.getLabelForValue(val))
}, [query]) }
useEffect(() => { return dateFormatter({
setTopStatData(null) interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
setTopStatsLoading(true) })(this.getLabelForValue(val))
setGraphData(null) },
setGraphLoading(true) color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
fetchTopStatsAndGraphData() }
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
if (isRealtime) { repositionTooltip(e) {
document.addEventListener('tick', fetchTopStatsAndGraphData) const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
} }
}
return () => { componentDidMount() {
document.removeEventListener('tick', fetchTopStatsAndGraphData) if (this.props.graphData) {
this.chart = this.regenerateChart();
} }
}, [query]) window.addEventListener('mousemove', this.repositionTooltip);
}
useEffect(() => { componentDidUpdate(prevProps) {
if (topStatData) { storeTopStatsContainerHeight() } const { graphData, darkTheme } = this.props;
}, [topStatData]) const tooltip = document.getElementById('chartjs-tooltip');
function fetchTopStatsAndGraphData() { if (
fetchTopStats(site, query) graphData !== prevProps.graphData ||
.then((res) => { darkTheme !== prevProps.darkTheme
setTopStatData(res) ) {
setTopStatsLoading(false)
})
let metric = getStoredMetric() if (graphData) {
const availableMetrics = getGraphableMetrics(query, site) if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (!availableMetrics.includes(metric)) { if (tooltip) {
metric = availableMetrics[0] tooltip.style.display = 'none';
storage.setItem(`metric__${site.domain}`, metric) }
} }
const interval = getCurrentInterval(site, query) if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
fetchGraphData(metric, interval) if (tooltip) {
tooltip.style.display = 'none';
}
}
} }
function fetchGraphData(metric, interval) { componentWillUnmount() {
fetchMainGraph(site, query, metric, interval) // Ensure that the tooltip doesn't hang around when we are loading more data
.then((res) => { const tooltip = document.getElementById('chartjs-tooltip');
setGraphData(res) if (tooltip) {
setGraphLoading(false) tooltip.style.opacity = 0;
setGraphRefreshing(false) tooltip.style.display = 'none';
}) }
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
} }
function getStoredMetric() { onClick(e) {
return storage.getItem(`metric__${site.domain}`) const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(this.pollExportReady.bind(this), 1000);
} else {
this.setState({ exported: false })
}
}
downloadSpinner() {
this.setState({ exported: true });
document.cookie = "exporting=";
setTimeout(this.pollExportReady.bind(this), 1000);
} }
function storeTopStatsContainerHeight() { downloadLink() {
storage.setItem(`topStatsHeight__${site.domain}`, document.getElementById('top-stats-container').clientHeight) if (this.props.query.period !== 'realtime') {
if (this.state.exported) {
return (
<div className="w-4 h-4 mx-2">
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
</div>
)
} else {
const interval = this.props.graphData?.interval
const queryParams = api.serializeQuery(this.props.query, [{ interval }])
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${queryParams}`
return (
<a className="w-4 h-4 mx-2" href={endpoint} download onClick={this.downloadSpinner.bind(this)}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
</a>
)
}
}
}
samplingNotice() {
const samplePercent = this.props.topStatData && this.props.topStatData.sample_percent
if (samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
}
}
importedNotice() {
if (!this.props.topStatData?.imported_source) return
const isBeforeNativeStats = (date) => {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(this.props.site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(this.props.topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
const strike = withImported ? "" : " line-through"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
<svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
</svg>
</div>
</Link>
)
}
} }
// This function is used for maintaining the main-graph/top-stats container height in the // This function is used for maintaining the main-graph/top-stats container height in the
// loading process. The container height depends on how many top stat metrics are returned // loading process. The container height depends on how many top stat metrics are returned
// from the API, but in the loading state, we don't know that yet. We can use localStorage // from the API, but in the loading state, we don't know that yet. We can use localStorage
// to keep track of the Top Stats container height. // to keep track of the Top Stats container height.
function getTopStatsHeight() { getTopStatsHeight() {
if (topStatData) { if (this.props.topStatData) {
return 'auto' return 'auto'
} else { } else {
return `${storage.getItem(`topStatsHeight__${site.domain}`) || 89}px` return `${storage.getItem(`topStatsHeight__${this.props.site.domain}`) || 89}px`
} }
} }
render() {
const { mainGraphRefreshing, updateMetric, updateInterval, metric, topStatData, query, site, graphData, lastLoadTimestamp } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return ( return (
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}> <div>
{(topStatsLoading || graphLoading) && renderLoader()} <div id="top-stats-container" className="flex flex-wrap" ref={this.boundary} style={{ height: this.getTopStatsHeight() }}>
<FadeIn show={!(topStatsLoading || graphLoading)}> <TopStats site={site} query={query} metric={metric} updateMetric={updateMetric} topStatData={topStatData} tooltipBoundary={this.boundary.current} lastLoadTimestamp={lastLoadTimestamp} />
<div id="top-stats-container" className="flex flex-wrap" ref={topStatsBoundary} style={{ height: getTopStatsHeight() }}>
<TopStats site={site} query={query} data={topStatData} onMetricUpdate={onMetricUpdate} tooltipBoundary={topStatsBoundary.current} lastLoadTimestamp={lastLoadTimestamp} />
</div> </div>
<div className="relative px-2"> <div className="relative px-2">
{graphRefreshing && renderLoader()} {mainGraphRefreshing && renderLoader()}
<div className="absolute right-4 -top-8 py-1 flex items-center"> <div className="absolute right-4 -top-8 py-1 flex items-center">
{!isRealtime && <StatsExport site={site} query={query} />} {this.downloadLink()}
<SamplingNotice samplePercent={topStatData}/> {this.samplingNotice()}
<WithImportedSwitch site={site} topStatData={topStatData} /> {this.importedNotice()}
<IntervalPicker site={site} query={query} onIntervalUpdate={onIntervalUpdate} /> <IntervalPicker site={site} query={query} graphData={graphData} metric={metric} updateInterval={updateInterval} />
</div> </div>
<LineGraphWithRouter graphData={graphData} darkTheme={isDarkTheme} query={query} /> <FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
</div> </div>
</div>
)
}
}
const LineGraphWithRouter = withRouter(LineGraph)
export default class VisitorGraph extends React.Component {
constructor(props) {
super(props)
this.state = {
topStatsLoadingState: LoadingState.loading,
mainGraphLoadingState: LoadingState.loading,
metric: storage.getItem(`metric__${this.props.site.domain}`) || 'visitors'
}
this.onVisible = this.onVisible.bind(this)
this.updateMetric = this.updateMetric.bind(this)
this.fetchTopStatData = this.fetchTopStatData.bind(this)
this.fetchGraphData = this.fetchGraphData.bind(this)
this.updateInterval = this.updateInterval.bind(this)
}
isIntervalValid(interval) {
const { query, site } = this.props
const validIntervals = site.validIntervalsByPeriod[query.period] || []
return validIntervals.includes(interval)
}
getIntervalFromStorage() {
const { query, site } = this.props
let interval = getStoredInterval(query.period, site.domain)
if (interval !== "week" && interval !== "month" && query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
interval = "month"
}
if (this.isIntervalValid(interval)) {
return interval
} else {
return null
}
}
updateInterval(interval) {
if (this.isIntervalValid(interval)) {
storeInterval(this.props.query.period, this.props.site.domain, interval)
this.setState({ mainGraphLoadingState: LoadingState.refreshing, graphData: null }, this.fetchGraphData)
}
}
onVisible() {
this.setState({ mainGraphLoadingState: LoadingState.loading }, this.fetchGraphData)
this.fetchTopStatData()
if (this.props.query.period === 'realtime') {
document.addEventListener('tick', this.fetchGraphData)
document.addEventListener('tick', this.fetchTopStatData)
}
}
componentDidUpdate(prevProps, prevState) {
const { metric } = this.state;
const { query } = this.props
if (query !== prevProps.query) {
this.setState({ mainGraphLoadingState: LoadingState.loading, topStatsLoadingState: LoadingState.loading, graphData: null, topStatData: null }, this.fetchGraphData)
this.fetchTopStatData()
}
if (metric !== prevState.metric) {
this.setState({ mainGraphLoadingState: LoadingState.refreshing }, this.fetchGraphData)
}
}
resetMetric() {
const { topStatData } = this.state
const { query, site } = this.props
const savedMetric = storage.getItem(`metric__${site.domain}`)
const selectableMetrics = topStatData && topStatData.top_stats.map(({ name }) => METRIC_MAPPING[name]).filter(name => name)
const canSelectSavedMetric = selectableMetrics && selectableMetrics.includes(savedMetric)
if (query.filters.goal && !['conversion_rate', 'events'].includes(savedMetric)) {
this.setState({ metric: 'conversions' })
} else if (canSelectSavedMetric) {
this.setState({ metric: savedMetric })
} else {
this.setState({ metric: 'visitors' })
}
}
componentWillUnmount() {
document.removeEventListener('tick', this.fetchGraphData)
document.removeEventListener('tick', this.fetchTopStatData)
}
storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${this.props.site.domain}`, document.getElementById('top-stats-container').clientHeight)
}
updateMetric(clickedMetric) {
if (this.state.metric == clickedMetric) return
storage.setItem(`metric__${this.props.site.domain}`, clickedMetric)
this.setState({ metric: clickedMetric, graphData: null })
}
fetchGraphData() {
const url = `/api/stats/${encodeURIComponent(this.props.site.domain)}/main-graph`
let params = { metric: this.state.metric }
const interval = this.getIntervalFromStorage()
if (interval) { params.interval = interval }
api.get(url, this.props.query, params)
.then((res) => {
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: res })
return res
})
.catch((err) => {
console.log(err)
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: false })
})
}
fetchTopStatData() {
const query = { ...this.props.query }
if (!isComparisonEnabled(query.comparison)) query.comparison = 'previous_period'
api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/top-stats`, query)
.then((res) => {
this.setState({ topStatsLoadingState: LoadingState.loaded, topStatData: res }, () => {
this.storeTopStatsContainerHeight()
this.resetMetric()
})
return res
})
}
renderInner() {
const { query, site } = this.props;
const { graphData, metric, topStatData, topStatsLoadingState, mainGraphLoadingState } = this.state;
const theme = document.querySelector('html').classList.contains('dark') || false
const mainGraphRefreshing = (mainGraphLoadingState === LoadingState.refreshing)
const topStatAndGraphLoaded = !!(topStatData && graphData)
const shouldShow =
topStatsLoadingState === LoadingState.loaded &&
LoadingState.isLoadedOrRefreshing(mainGraphLoadingState) &&
(topStatData && mainGraphRefreshing || topStatAndGraphLoaded)
return (
<FadeIn show={shouldShow}>
<LineGraphWithRouter mainGraphRefreshing={mainGraphRefreshing} graphData={graphData} topStatData={topStatData} site={site} query={query} darkTheme={theme} metric={metric} updateMetric={this.updateMetric} updateInterval={this.updateInterval} lastLoadTimestamp={this.props.lastLoadTimestamp} />
</FadeIn> </FadeIn>
</div> )
) }
render() {
const { mainGraphLoadingState, topStatsLoadingState } = this.state
const showLoader =
[mainGraphLoadingState, topStatsLoadingState].includes(LoadingState.loading) &&
mainGraphLoadingState !== LoadingState.refreshing
return (
<LazyLoader onVisible={this.onVisible}>
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{showLoader && renderLoader()}
{this.renderInner()}
</div>
</LazyLoader>
)
}
} }
function renderLoader() { function renderLoader() {

@ -1,40 +0,0 @@
import React from "react"
import { parseNaiveDate, isBefore } from '../../util/date'
import { Link } from 'react-router-dom'
import * as url from '../../util/url'
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
export default function WithImportedSwitch({site, topStatData}) {
if (!topStatData?.imports_exist) {
return null
}
function isBeforeNativeStats(date) {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const withImported = topStatData.with_imported;
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
<BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)
} else {
return null
}
}

@ -1,5 +1,3 @@
const flags = JSON.parse(document.getElementById('stats-react-container').dataset.flags)
export const FILTER_GROUPS = { export const FILTER_GROUPS = {
'page': ['page', 'entry_page', 'exit_page'], 'page': ['page', 'entry_page', 'exit_page'],
'source': ['source', 'referrer'], 'source': ['source', 'referrer'],
@ -9,9 +7,7 @@ export const FILTER_GROUPS = {
'os': ['os', 'os_version'], 'os': ['os', 'os_version'],
'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'], 'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'],
'goal': ['goal'], 'goal': ['goal'],
'props': ['prop_key', 'prop_value'], 'props': ['prop_key', 'prop_value']
...(flags.hostname_filter ? { 'hostname': ['hostname'] } : {})
} }
export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location'])) export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location']))
@ -41,7 +37,7 @@ export function isFreeChoiceFilter(filterName) {
let NON_ESCAPED_PIPE_REGEX; let NON_ESCAPED_PIPE_REGEX;
try { try {
NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g") NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g")
} catch (_e) { } catch(_e) {
NON_ESCAPED_PIPE_REGEX = '|' NON_ESCAPED_PIPE_REGEX = '|'
} }
@ -68,19 +64,19 @@ export function parsePrefix(rawValue) {
.filter((clause) => !!clause) .filter((clause) => !!clause)
.map((val) => val.replaceAll(ESCAPED_PIPE, '|')) .map((val) => val.replaceAll(ESCAPED_PIPE, '|'))
return { type, values } return {type, values}
} }
export function parseQueryPropsFilter(query) { export function parseQueryPropsFilter(query) {
return Object.entries(query.filters['props']).map(([key, propVal]) => { return Object.entries(query.filters['props']).map(([key, propVal]) => {
const { type, values } = parsePrefix(propVal) const {type, values} = parsePrefix(propVal)
const clauses = values.map(val => { return { value: val, label: val } }) const clauses = values.map(val => { return {value: val, label: val}})
return { propKey: { label: key, value: key }, type, clauses } return { propKey: { label: key, value: key }, type, clauses }
}) })
} }
export function parseQueryFilter(query, filter) { export function parseQueryFilter(query, filter) {
const { type, values } = parsePrefix(query.filters[filter] || '') const {type, values} = parsePrefix(query.filters[filter] || '')
let labels = values let labels = values
@ -99,9 +95,9 @@ export function parseQueryFilter(query, filter) {
labels = rawLabel.split('|').filter(label => !!label) labels = rawLabel.split('|').filter(label => !!label)
} }
const clauses = values.map((value, index) => { return { value, label: labels[index] } }) const clauses = values.map((value, index) => { return {value, label: labels[index]}})
return { type, clauses } return {type, clauses}
} }
export function isFilteringOnFixedValue(query, filter) { export function isFilteringOnFixedValue(query, filter) {
@ -156,7 +152,6 @@ export const formattedFilters = {
'region': 'Region', 'region': 'Region',
'city': 'City', 'city': 'City',
'page': 'Page', 'page': 'Page',
'hostname': 'Hostname',
'entry_page': 'Entry Page', 'entry_page': 'Entry Page',
'exit_page': 'Exit Page', 'exit_page': 'Exit Page'
} }

@ -19,30 +19,12 @@ if (csrfToken && websocketUrl) {
}) })
} }
} }
let Uploaders = {}
Uploaders.S3 = function (entries, onViewError) {
entries.forEach(entry => {
let xhr = new XMLHttpRequest()
onViewError(() => xhr.abort())
xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
xhr.onerror = () => entry.error()
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
let percent = Math.round((event.loaded / event.total) * 100)
if (percent < 100) { entry.progress(percent) }
}
})
let url = entry.meta.url
xhr.open("PUT", url, true)
xhr.send(entry.file)
})
}
let token = csrfToken.getAttribute("content") let token = csrfToken.getAttribute("content")
let url = websocketUrl.getAttribute("content") let url = websocketUrl.getAttribute("content")
let liveUrl = (url === "") ? "/live" : new URL("/live", url).href; let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
let liveSocket = new LiveSocket(liveUrl, Socket, { let liveSocket = new LiveSocket(liveUrl, Socket, {
heartbeatIntervalMs: 10000, heartbeatIntervalMs: 10000,
params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: { params: { _csrf_token: token }, hooks: Hooks, dom: {
// for alpinejs integration // for alpinejs integration
onBeforeElUpdated(from, to) { onBeforeElUpdated(from, to) {
if (from._x_dataStack) { if (from._x_dataStack) {

@ -38,7 +38,7 @@ config :tailwind,
config :ua_inspector, config :ua_inspector,
database_path: "priv/ua_inspector", database_path: "priv/ua_inspector",
remote_release: "6.3.0" remote_release: "6.2.1"
config :ref_inspector, config :ref_inspector,
database_path: "priv/ref_inspector" database_path: "priv/ref_inspector"

@ -1,38 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240302"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -10,7 +10,7 @@
"kind": "analyticsData#runReport", "kind": "analyticsData#runReport",
"metadata": { "metadata": {
"currencyCode": "USD", "currencyCode": "USD",
"timeZone": "Europe/Amsterdam" "timeZone": "Europe/Warsaw"
}, },
"metricHeaders": [ "metricHeaders": [
{ {
@ -34,749 +34,77 @@
"type": "TYPE_SECONDS" "type": "TYPE_SECONDS"
} }
], ],
"rowCount": 31, "rowCount": 3,
"rows": [ "rows": [
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240131" "value": "20240226"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "141" "value": "1"
}, },
{ {
"value": "195" "value": "1"
}, },
{ {
"value": "56" "value": "1"
}, },
{ {
"value": "146" "value": "1"
}, },
{ {
"value": "4815" "value": "0"
} }
] ]
}, },
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240130" "value": "20240224"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "115" "value": "2"
}, },
{ {
"value": "171" "value": "2"
}, },
{ {
"value": "42" "value": "2"
}, },
{ {
"value": "115" "value": "2"
}, },
{ {
"value": "3481" "value": "0"
} }
] ]
}, },
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240129" "value": "20240222"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "105" "value": "3"
}, },
{ {
"value": "153" "value": "13"
}, },
{ {
"value": "54" "value": "0"
}, },
{ {
"value": "113" "value": "4"
}, },
{ {
"value": "2895" "value": "29"
}
]
},
{
"dimensionValues": [
{
"value": "20240128"
}
],
"metricValues": [
{
"value": "184"
},
{
"value": "259"
},
{
"value": "74"
},
{
"value": "190"
},
{
"value": "6861"
}
]
},
{
"dimensionValues": [
{
"value": "20240127"
}
],
"metricValues": [
{
"value": "155"
},
{
"value": "202"
},
{
"value": "67"
},
{
"value": "158"
},
{
"value": "5259"
}
]
},
{
"dimensionValues": [
{
"value": "20240126"
}
],
"metricValues": [
{
"value": "132"
},
{
"value": "189"
},
{
"value": "57"
},
{
"value": "142"
},
{
"value": "5545"
}
]
},
{
"dimensionValues": [
{
"value": "20240125"
}
],
"metricValues": [
{
"value": "151"
},
{
"value": "229"
},
{
"value": "71"
},
{
"value": "155"
},
{
"value": "5365"
}
]
},
{
"dimensionValues": [
{
"value": "20240124"
}
],
"metricValues": [
{
"value": "137"
},
{
"value": "221"
},
{
"value": "64"
},
{
"value": "148"
},
{
"value": "4454"
}
]
},
{
"dimensionValues": [
{
"value": "20240123"
}
],
"metricValues": [
{
"value": "166"
},
{
"value": "267"
},
{
"value": "61"
},
{
"value": "171"
},
{
"value": "4850"
}
]
},
{
"dimensionValues": [
{
"value": "20240122"
}
],
"metricValues": [
{
"value": "161"
},
{
"value": "233"
},
{
"value": "73"
},
{
"value": "165"
},
{
"value": "4871"
}
]
},
{
"dimensionValues": [
{
"value": "20240121"
}
],
"metricValues": [
{
"value": "254"
},
{
"value": "397"
},
{
"value": "114"
},
{
"value": "267"
},
{
"value": "8845"
}
]
},
{
"dimensionValues": [
{
"value": "20240120"
}
],
"metricValues": [
{
"value": "214"
},
{
"value": "383"
},
{
"value": "100"
},
{
"value": "223"
},
{
"value": "6914"
}
]
},
{
"dimensionValues": [
{
"value": "20240119"
}
],
"metricValues": [
{
"value": "381"
},
{
"value": "5571"
},
{
"value": "100"
},
{
"value": "228"
},
{
"value": "98549"
}
]
},
{
"dimensionValues": [
{
"value": "20240118"
}
],
"metricValues": [
{
"value": "177"
},
{
"value": "278"
},
{
"value": "86"
},
{
"value": "186"
},
{
"value": "5553"
}
]
},
{
"dimensionValues": [
{
"value": "20240117"
}
],
"metricValues": [
{
"value": "204"
},
{
"value": "271"
},
{
"value": "97"
},
{
"value": "216"
},
{
"value": "6777"
}
]
},
{
"dimensionValues": [
{
"value": "20240116"
}
],
"metricValues": [
{
"value": "200"
},
{
"value": "280"
},
{
"value": "65"
},
{
"value": "203"
},
{
"value": "9381"
}
]
},
{
"dimensionValues": [
{
"value": "20240115"
}
],
"metricValues": [
{
"value": "255"
},
{
"value": "311"
},
{
"value": "86"
},
{
"value": "269"
},
{
"value": "12365"
}
]
},
{
"dimensionValues": [
{
"value": "20240114"
}
],
"metricValues": [
{
"value": "243"
},
{
"value": "302"
},
{
"value": "86"
},
{
"value": "257"
},
{
"value": "11073"
}
]
},
{
"dimensionValues": [
{
"value": "20240113"
}
],
"metricValues": [
{
"value": "231"
},
{
"value": "276"
},
{
"value": "92"
},
{
"value": "244"
},
{
"value": "10084"
}
]
},
{
"dimensionValues": [
{
"value": "20240112"
}
],
"metricValues": [
{
"value": "215"
},
{
"value": "265"
},
{
"value": "75"
},
{
"value": "225"
},
{
"value": "10195"
}
]
},
{
"dimensionValues": [
{
"value": "20240111"
}
],
"metricValues": [
{
"value": "183"
},
{
"value": "235"
},
{
"value": "64"
},
{
"value": "191"
},
{
"value": "8200"
}
]
},
{
"dimensionValues": [
{
"value": "20240110"
}
],
"metricValues": [
{
"value": "190"
},
{
"value": "244"
},
{
"value": "61"
},
{
"value": "203"
},
{
"value": "9056"
}
]
},
{
"dimensionValues": [
{
"value": "20240109"
}
],
"metricValues": [
{
"value": "201"
},
{
"value": "290"
},
{
"value": "63"
},
{
"value": "213"
},
{
"value": "11651"
}
]
},
{
"dimensionValues": [
{
"value": "20240108"
}
],
"metricValues": [
{
"value": "195"
},
{
"value": "235"
},
{
"value": "71"
},
{
"value": "209"
},
{
"value": "7946"
}
]
},
{
"dimensionValues": [
{
"value": "20240107"
}
],
"metricValues": [
{
"value": "205"
},
{
"value": "274"
},
{
"value": "80"
},
{
"value": "218"
},
{
"value": "10048"
}
]
},
{
"dimensionValues": [
{
"value": "20240106"
}
],
"metricValues": [
{
"value": "256"
},
{
"value": "303"
},
{
"value": "119"
},
{
"value": "281"
},
{
"value": "8440"
}
]
},
{
"dimensionValues": [
{
"value": "20240105"
}
],
"metricValues": [
{
"value": "226"
},
{
"value": "270"
},
{
"value": "83"
},
{
"value": "237"
},
{
"value": "9311"
}
]
},
{
"dimensionValues": [
{
"value": "20240104"
}
],
"metricValues": [
{
"value": "223"
},
{
"value": "274"
},
{
"value": "81"
},
{
"value": "236"
},
{
"value": "12051"
}
]
},
{
"dimensionValues": [
{
"value": "20240103"
}
],
"metricValues": [
{
"value": "233"
},
{
"value": "283"
},
{
"value": "79"
},
{
"value": "245"
},
{
"value": "10628"
}
]
},
{
"dimensionValues": [
{
"value": "20240102"
}
],
"metricValues": [
{
"value": "203"
},
{
"value": "269"
},
{
"value": "60"
},
{
"value": "210"
},
{
"value": "11168"
}
]
},
{
"dimensionValues": [
{
"value": "20240101"
}
],
"metricValues": [
{
"value": "191"
},
{
"value": "224"
},
{
"value": "71"
},
{
"value": "197"
},
{
"value": "8080"
} }
] ]
} }

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20200421"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20170118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -24,7 +24,6 @@ defmodule Mix.Tasks.SendPageview do
page: :string, page: :string,
referrer: :string, referrer: :string,
host: :string, host: :string,
hostname: :string,
event: :string, event: :string,
props: :string, props: :string,
revenue_currency: :string, revenue_currency: :string,
@ -87,7 +86,6 @@ defmodule Mix.Tasks.SendPageview do
referrer = Keyword.get(opts, :referrer, @default_referrer) referrer = Keyword.get(opts, :referrer, @default_referrer)
event = Keyword.get(opts, :event, @default_event) event = Keyword.get(opts, :event, @default_event)
props = Keyword.get(opts, :props, @default_props) props = Keyword.get(opts, :props, @default_props)
hostname = Keyword.get(opts, :hostname, domain)
revenue = revenue =
if Keyword.get(opts, :revenue_currency) do if Keyword.get(opts, :revenue_currency) do
@ -99,7 +97,7 @@ defmodule Mix.Tasks.SendPageview do
%{ %{
name: event, name: event,
url: "http://#{hostname}#{page}", url: "http://#{domain}#{page}",
domain: domain, domain: domain,
referrer: referrer, referrer: referrer,
props: props, props: props,

@ -77,21 +77,6 @@ defmodule Plausible.Application do
interval: :timer.seconds(35), interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently warmer_fn: :refresh_updated_recently
]}, ]},
{Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.All,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
PlausibleWeb.Endpoint, PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)}, {Oban, Application.get_env(:plausible, Oban)},

@ -176,16 +176,26 @@ defmodule Plausible.Cache do
defp refresh(mode, query, opts) when mode in @modes do defp refresh(mode, query, opts) when mode in @modes do
cache_name = Keyword.get(opts, :cache_name, name()) cache_name = Keyword.get(opts, :cache_name, name())
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( measure_duration(telemetry_event_refresh(cache_name, mode), fn ->
telemetry_event_refresh(cache_name, mode), items = Plausible.Repo.all(query)
fn -> :ok = merge_items(items, opts)
items = Plausible.Repo.all(query) end)
:ok = merge_items(items, opts)
end
)
:ok :ok
end end
defp measure_duration(event, fun) when is_function(fun, 0) do
{duration, result} = time_it(fun)
:telemetry.execute(event, %{duration: duration}, %{})
result
end
defp time_it(fun) do
start = System.monotonic_time()
result = fun.()
stop = System.monotonic_time()
{stop - start, result}
end
end end
end end
end end

@ -65,28 +65,4 @@ defmodule Plausible.ClickhouseEventV2 do
) )
|> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp]) |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
end end
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
def merge_session(%__MODULE__{} = event, session) do
Map.merge(event, Map.take(session, @session_properties))
end
end end

@ -43,7 +43,6 @@ defmodule Plausible.ClickhouseSessionV2 do
field :is_bounce, BoolUInt8 field :is_bounce, BoolUInt8
field :entry_page, :string field :entry_page, :string
field :exit_page, :string field :exit_page, :string
field :exit_page_hostname, :string
field :pageviews, Ch, type: "Int32" field :pageviews, Ch, type: "Int32"
field :events, Ch, type: "Int32" field :events, Ch, type: "Int32"
field :sign, Ch, type: "Int8" field :sign, Ch, type: "Int8"

@ -1,188 +0,0 @@
defmodule Plausible.DataMigration.SiteImports do
@moduledoc """
Site imports migration backfilling SiteImport entries for old imports
and alters import end dates to match actual end date of respective import stats.
"""
import Ecto.Query
alias Plausible.ClickhouseRepo
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
def run(opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
sites_with_imports =
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|> Repo.all(log: false)
sites_count = length(sites_with_imports)
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
for {site, idx} <- Enum.with_index(sites_with_imports) do
site_imports =
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|> Repo.all(log: false)
IO.puts(
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
)
site_imports =
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
IO.puts("Creating legacy site import entry for site ID #{site.id}")
# create legacy entry if there's not one yet
params =
site.imported_data
|> Imported.SiteImport.from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
legacy_site_import =
%Imported.SiteImport{}
|> Ecto.Changeset.change(params)
|> insert!(dry_run?)
[legacy_site_import | site_imports]
else
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
site_imports
end
# adjust end date for each site import
for site_import <- site_imports do
IO.puts(
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
)
import_ids =
if site_import.legacy do
[0, site_import.id]
else
[site_import.id]
end
end_date = imported_stats_end_date(site.id, import_ids)
if !end_date do
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
)
if site_import.legacy do
clear_imported_data(site, dry_run?)
end
delete!(site_import, dry_run?)
else
end_date =
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
end_date
else
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
)
site_import.end_date
end
site_import
|> Ecto.Changeset.change(end_date: end_date)
|> update!(dry_run?)
IO.puts(
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
)
end
end
IO.puts("Done processing site ID #{site.id}")
end
IO.puts("Finished")
end
# Exposed for testing purposes
@doc false
def imported_stats_end_date(site_id, import_ids) do
[first_schema | schemas] = Imported.schemas()
query =
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
end)
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
if dates != [] do
case Enum.max(dates, Date) do
# no stats for this domain yet
~D[1970-01-01] ->
nil
date ->
date
end
else
nil
end
end
defp insert!(changeset, false = _dry_run?) do
Repo.insert!(changeset)
end
defp insert!(changeset, true = _dry_run?) do
if changeset.valid? do
changeset
|> Ecto.Changeset.change(id: 0)
|> Ecto.Changeset.apply_changes()
else
raise "Invalid insert: #{inspect(changeset)}"
end
end
defp clear_imported_data(site, false = _dry_run?) do
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
end
defp clear_imported_data(site, true = _dry_run?) do
%{site | imported_data: nil}
end
defp update!(changeset, false = _dry_run?) do
Repo.update!(changeset)
end
defp update!(changeset, true = _dry_run?) do
if changeset.valid? do
Ecto.Changeset.apply_changes(changeset)
else
raise "Invalid update: #{inspect(changeset)}"
end
end
defp delete!(entity, false = _dry_run?) do
Repo.delete!(entity)
end
defp delete!(entity, true = _dry_run?) do
entity
end
defp max_date_query(schema, site_id, import_ids) do
from(q in schema,
where: q.site_id == ^site_id,
where: q.import_id in ^import_ids,
select: %{max_date: fragment("max(?)", q.date)}
)
end
end

@ -12,24 +12,14 @@ defmodule Plausible.Exports do
Examples: Examples:
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31]) iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
"plausible_io_20210101_20241231.zip" "plausible.io_20210101_20241231.zip"
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31]) iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
"Bücher_example_20210101_20241231.zip" "Bücher.example_20210101_20241231.zip"
""" """
def archive_filename(domain, min_date, max_date) do def archive_filename(domain, min_date, max_date) do
name = "#{domain}_#{Calendar.strftime(min_date, "%Y%m%d")}_#{Calendar.strftime(max_date, "%Y%m%d")}.zip"
Enum.join(
[
String.replace(domain, ".", "_"),
Calendar.strftime(min_date, "%Y%m%d"),
Calendar.strftime(max_date, "%Y%m%d")
],
"_"
)
name <> ".zip"
end end
@doc """ @doc """
@ -118,27 +108,51 @@ defmodule Plausible.Exports do
end end
end end
defmacrop pageviews(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
:pageviews
)
end
end
@spec export_visitors_q(pos_integer) :: Ecto.Query.t() @spec export_visitors_q(pos_integer) :: Ecto.Query.t()
def export_visitors_q(site_id) do def export_visitors_q(site_id) do
from s in sampled("sessions_v2"), visitors_sessions_q =
where: s.site_id == ^site_id, from s in sampled("sessions_v2"),
group_by: selected_as(:date), where: s.site_id == ^site_id,
group_by: selected_as(:date),
select: %{
date: date(s.start),
bounces: bounces(s),
visits: visits(s),
visit_duration: visit_duration(s)
# NOTE: can we use just sessions_v2 table in this query? sum(pageviews) and visitors(s)?
# visitors: visitors(s)
}
visitors_events_q =
from e in sampled("events_v2"),
where: e.site_id == ^site_id,
group_by: selected_as(:date),
select: %{
date: date(e.timestamp),
visitors: visitors(e),
pageviews:
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
:pageviews
)
}
visitors_q =
"e"
|> with_cte("e", as: ^visitors_events_q)
|> with_cte("s", as: ^visitors_sessions_q)
from e in visitors_q,
full_join: s in "s",
on: e.date == s.date,
order_by: selected_as(:date),
select: [ select: [
date(s.start), selected_as(fragment("greatest(?,?)", s.date, e.date), :date),
visitors(s), e.visitors,
pageviews(s), e.pageviews,
bounces(s), s.bounces,
visits(s), s.visits,
visit_duration(s) s.visit_duration
] ]
end end
@ -149,8 +163,6 @@ defmodule Plausible.Exports do
group_by: [ group_by: [
selected_as(:date), selected_as(:date),
selected_as(:source), selected_as(:source),
s.referrer,
s.utm_source,
s.utm_medium, s.utm_medium,
s.utm_campaign, s.utm_campaign,
s.utm_content, s.utm_content,
@ -160,13 +172,10 @@ defmodule Plausible.Exports do
select: [ select: [
date(s.start), date(s.start),
selected_as(s.referrer_source, :source), selected_as(s.referrer_source, :source),
s.referrer,
s.utm_source,
s.utm_medium, s.utm_medium,
s.utm_campaign, s.utm_campaign,
s.utm_content, s.utm_content,
s.utm_term, s.utm_term,
pageviews(s),
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
@ -191,7 +200,6 @@ defmodule Plausible.Exports do
hostname: e.hostname, hostname: e.hostname,
name: e.name, name: e.name,
user_id: e.user_id, user_id: e.user_id,
session_id: e.session_id,
_sample_factor: fragment("_sample_factor") _sample_factor: fragment("_sample_factor")
} }
@ -202,10 +210,6 @@ defmodule Plausible.Exports do
date(e.timestamp), date(e.timestamp),
selected_as(fragment("any(?)", e.hostname), :hostname), selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page), selected_as(e.pathname, :page),
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
:visits
),
visitors(e), visitors(e),
selected_as( selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name), fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
@ -238,8 +242,7 @@ defmodule Plausible.Exports do
:entrances :entrances
), ),
visit_duration(s), visit_duration(s),
bounces(s), bounces(s)
pageviews(s)
] ]
end end
@ -253,13 +256,10 @@ defmodule Plausible.Exports do
date(s.start), date(s.start),
s.exit_page, s.exit_page,
visitors(s), visitors(s),
visit_duration(s),
selected_as( selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign), fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:exits :exits
), )
bounces(s),
pageviews(s)
] ]
end end
@ -278,8 +278,7 @@ defmodule Plausible.Exports do
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s), bounces(s)
pageviews(s)
] ]
end end
@ -295,8 +294,7 @@ defmodule Plausible.Exports do
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s), bounces(s)
pageviews(s)
] ]
end end
@ -304,17 +302,15 @@ defmodule Plausible.Exports do
def export_browsers_q(site_id) do def export_browsers_q(site_id) do
from s in sampled("sessions_v2"), from s in sampled("sessions_v2"),
where: s.site_id == ^site_id, where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.browser, s.browser_version], group_by: [selected_as(:date), s.browser],
order_by: selected_as(:date), order_by: selected_as(:date),
select: [ select: [
date(s.start), date(s.start),
s.browser, s.browser,
s.browser_version,
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s), bounces(s)
pageviews(s)
] ]
end end
@ -322,17 +318,15 @@ defmodule Plausible.Exports do
def export_operating_systems_q(site_id) do def export_operating_systems_q(site_id) do
from s in sampled("sessions_v2"), from s in sampled("sessions_v2"),
where: s.site_id == ^site_id, where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.operating_system, s.operating_system_version], group_by: [selected_as(:date), s.operating_system],
order_by: selected_as(:date), order_by: selected_as(:date),
select: [ select: [
date(s.start), date(s.start),
s.operating_system, s.operating_system,
s.operating_system_version,
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s), bounces(s)
pageviews(s)
] ]
end end

@ -80,14 +80,6 @@ defmodule Plausible.Goals do
def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"} def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"}
def list_revenue_goals(site) do
from(g in Plausible.Goal,
where: g.site_id == ^site.id and not is_nil(g.currency),
select: %{event_name: g.event_name, currency: g.currency}
)
|> Plausible.Repo.all()
end
def for_site(site, opts \\ []) do def for_site(site, opts \\ []) do
site site
|> for_site_query(opts) |> for_site_query(opts)

@ -55,14 +55,6 @@ defmodule Plausible.Google.API do
end end
end end
def get_analytics_end_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view)
end
end
def fetch_verified_properties(auth) do def fetch_verified_properties(auth) do
with {:ok, access_token} <- maybe_refresh_token(auth), with {:ok, access_token} <- maybe_refresh_token(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do

@ -64,10 +64,6 @@ defmodule Plausible.Google.GA4.API do
GA4.HTTP.get_analytics_start_date(access_token, property) GA4.HTTP.get_analytics_start_date(access_token, property)
end end
def get_analytics_end_date(access_token, property) do
GA4.HTTP.get_analytics_end_date(access_token, property)
end
def import_analytics(date_range, property, auth, persist_fn) do def import_analytics(date_range, property, auth, persist_fn) do
Logger.debug( Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"

@ -140,11 +140,8 @@ defmodule Plausible.Google.GA4.HTTP do
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed} {:error, :authentication_failed}
{:error, %{reason: :timeout}} -> {:error, %HTTPClient.Non200Error{} = error} ->
{:error, :timeout} Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error})
{:error, error} ->
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
{:error, :unknown} {:error, :unknown}
end end
end end
@ -164,11 +161,8 @@ defmodule Plausible.Google.GA4.HTTP do
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] -> {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
{:error, :not_found} {:error, :not_found}
{:error, %{reason: :timeout}} -> {:error, %HTTPClient.Non200Error{} = error} ->
{:error, :timeout} Sentry.capture_message("Error retrieving Google property #{property}",
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 property #{property}",
extra: %{error: error} extra: %{error: error}
) )
@ -177,18 +171,7 @@ defmodule Plausible.Google.GA4.HTTP do
end end
@earliest_valid_date "2015-08-14" @earliest_valid_date "2015-08-14"
def get_analytics_start_date(access_token, property) do def get_analytics_start_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :start)
end
def get_analytics_end_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :end)
end
defp get_analytics_boundary_date(access_token, property, edge) do
descending? = edge == :end
params = %{ params = %{
requests: [ requests: [
%{ %{
@ -199,7 +182,7 @@ defmodule Plausible.Google.GA4.HTTP do
dimensions: [%{name: "date"}], dimensions: [%{name: "date"}],
metrics: [%{name: "screenPageViews"}], metrics: [%{name: "screenPageViews"}],
orderBys: [ orderBys: [
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?} %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false}
], ],
limit: 1 limit: 1
} }
@ -224,18 +207,13 @@ defmodule Plausible.Google.GA4.HTTP do
{:ok, date} {:ok, date}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> {:error, %{reason: %Finch.Response{body: body}}} ->
{:error, :authentication_failed} Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: :timeout}} -> {:error, %{reason: reason} = e} ->
{:error, :timeout} Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)})
{:error, reason}
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 #{edge} date",
extra: %{error: error}
)
{:error, :unknown}
end end
end end

@ -46,11 +46,9 @@ defmodule Plausible.Google.GA4.ReportRequest do
"sessionMedium", "sessionMedium",
"sessionCampaignName", "sessionCampaignName",
"sessionManualAdContent", "sessionManualAdContent",
"sessionGoogleAdsKeyword", "sessionGoogleAdsKeyword"
"pageReferrer"
], ],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -61,13 +59,12 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_pages", dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"], dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric # NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"] metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"]
}, },
%__MODULE__{ %__MODULE__{
dataset: "imported_entry_pages", dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"], dimensions: ["date", "landingPage"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"userEngagementDuration", "userEngagementDuration",
@ -78,19 +75,12 @@ defmodule Plausible.Google.GA4.ReportRequest do
# %__MODULE__{ # %__MODULE__{
# dataset: "imported_exit_pages", # dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"], # dimensions: ["date", "ga:exitPagePath"],
# metrics: [ # metrics: ["totalUsers", "sessions"]
# "totalUsers",
# "sessions",
# "screenPageViews",
# "userEngagementDuration",
# "bounces = sessions - engagedSessions"
# ]
# }, # },
%__MODULE__{ %__MODULE__{
dataset: "imported_locations", dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"], dimensions: ["date", "countryId", "region", "city"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -101,7 +91,6 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_devices", dataset: "imported_devices",
dimensions: ["date", "deviceCategory"], dimensions: ["date", "deviceCategory"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -112,7 +101,6 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_browsers", dataset: "imported_browsers",
dimensions: ["date", "browser"], dimensions: ["date", "browser"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -121,9 +109,8 @@ defmodule Plausible.Google.GA4.ReportRequest do
}, },
%__MODULE__{ %__MODULE__{
dataset: "imported_operating_systems", dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem", "operatingSystemVersion"], dimensions: ["date", "operatingSystem"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",

@ -64,10 +64,6 @@ defmodule Plausible.Google.UA.API do
UA.HTTP.get_analytics_start_date(access_token, view_id) UA.HTTP.get_analytics_start_date(access_token, view_id)
end end
def get_analytics_end_date(access_token, view_id) do
UA.HTTP.get_analytics_end_date(access_token, view_id)
end
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()} :ok | {:error, term()}
@doc """ @doc """

@ -108,33 +108,14 @@ defmodule Plausible.Google.UA.HTTP do
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed} {:error, :authentication_failed}
{:error, %{reason: :timeout}} -> {:error, %HTTPClient.Non200Error{} = error} ->
{:error, :timeout} Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
{:error, error} ->
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
{:error, :unknown} {:error, :unknown}
end end
end end
@earliest_valid_date "2005-01-01" @earliest_valid_date "2005-01-01"
def get_analytics_start_date(access_token, view_id) do def get_analytics_start_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :start)
end
def get_analytics_end_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :end)
end
defp get_analytics_boundary_date(access_token, view_id, edge) do
sort_order =
if edge == :start do
"ASCENDING"
else
"DESCENDING"
end
params = %{ params = %{
reportRequests: [ reportRequests: [
%{ %{
@ -146,7 +127,7 @@ defmodule Plausible.Google.UA.HTTP do
metrics: [%{expression: "ga:pageviews"}], metrics: [%{expression: "ga:pageviews"}],
hideTotals: true, hideTotals: true,
hideValueRanges: true, hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}], orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
pageSize: 1 pageSize: 1
} }
] ]
@ -170,18 +151,13 @@ defmodule Plausible.Google.UA.HTTP do
{:ok, date} {:ok, date}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> {:error, %{reason: %Finch.Response{body: body}}} ->
{:error, :authentication_failed} Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving UA #{edge} date",
extra: %{error: error}
)
{:error, :unknown} {:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)})
{:error, reason}
end end
end end

@ -37,9 +37,6 @@ defmodule Plausible.Imported do
# Maximum number of complete imports to account for when querying stats # Maximum number of complete imports to account for when querying stats
@max_complete_imports 5 @max_complete_imports 5
@spec schemas() :: [module()]
def schemas, do: @tables
@spec tables() :: [String.t()] @spec tables() :: [String.t()]
def tables, do: @table_names def tables, do: @table_names
@ -51,29 +48,28 @@ defmodule Plausible.Imported do
def load_import_data(site) do def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site) complete_import_ids = list_complete_import_ids(site)
dates = get_imports_date_range(site) earliest_import = get_earliest_import(site) || %{}
%{ %{
site site
| import_data_loaded: true, | import_data_loaded: true,
earliest_import_start_date: dates.start_date, earliest_import_start_date: Map.get(earliest_import, :start_date),
latest_import_end_date: dates.end_date, earliest_import_end_date: Map.get(earliest_import, :end_date),
complete_import_ids: complete_import_ids complete_import_ids: complete_import_ids
} }
end end
@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil @spec get_import(non_neg_integer()) :: SiteImport.t() | nil
def get_import(site, import_id) do def get_import(import_id) do
Repo.get_by(SiteImport, id: import_id, site_id: site.id) Repo.get(SiteImport, import_id)
end end
defdelegate listen(), to: Imported.Importer defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()] @spec list_all_imports(Site.t()) :: [SiteImport.t()]
def list_all_imports(site, status \\ nil) do def list_all_imports(site) do
imports = imports =
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at]) from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|> maybe_filter_by_status(status)
|> Repo.all() |> Repo.all()
if site.imported_data && not Enum.any?(imports, & &1.legacy) do if site.imported_data && not Enum.any?(imports, & &1.legacy) do
@ -83,72 +79,42 @@ defmodule Plausible.Imported do
end end
end end
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
def other_imports_in_progress?(site_import) do
Repo.exists?(
from(i in SiteImport,
where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
)
)
end
defp maybe_filter_by_status(query, nil), do: query
defp maybe_filter_by_status(query, status) do
where(query, [i], i.status == ^status)
end
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()] @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
def list_complete_import_ids(site) do def list_complete_import_ids(site) do
ids = ids =
from(i in SiteImport, from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(), where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: {i.legacy, i.id}, select: i.id,
limit: @max_complete_imports limit: @max_complete_imports
) )
|> Repo.all() |> Repo.all()
has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)
ids = Enum.map(ids, &elem(&1, 1))
# account for legacy imports as well # account for legacy imports as well
if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do if site.imported_data && site.imported_data.status == "ok" do
[0 | ids] [0 | ids]
else else
ids ids
end end
end end
@spec get_imports_date_range(Site.t()) :: %{ @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
start_date: Date.t() | nil, def get_earliest_import(site) do
end_date: Date.t() | nil first_import =
}
def get_imports_date_range(site) do
dates =
from(i in SiteImport, from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(), where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: %{start_date: min(i.start_date), end_date: max(i.end_date)} order_by: i.start_date,
limit: 1
) )
|> Repo.one() |> Repo.one()
dates = dates || %{start_date: nil, end_date: nil} legacy_import =
if site.imported_data && site.imported_data.status == "ok" do
if site.imported_data && site.imported_data.status == "ok" do site.imported_data
start_date = end
[dates.start_date, site.imported_data.start_date]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
end_date =
[dates.end_date, site.imported_data.end_date]
|> Enum.reject(&is_nil/1)
|> Enum.max(Date, fn -> nil end)
%{start_date: start_date, end_date: end_date} [legacy_import, first_import]
else |> Enum.reject(&is_nil/1)
dates |> Enum.min_by(& &1.start_date, Date, fn -> nil end)
end
end end
@spec delete_imports_for_site(Site.t()) :: :ok @spec delete_imports_for_site(Site.t()) :: :ok
@ -157,66 +123,4 @@ defmodule Plausible.Imported do
:ok :ok
end end
@spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) ::
{:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window}
def check_dates(_site, nil, _end_date), do: {:error, :no_data}
def check_dates(site, start_date, end_date) do
cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
end_date = Enum.min([end_date, cutoff_date], Date)
with true <- Date.diff(end_date, start_date) >= 2,
[_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do
longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first))
{:ok, longest.first, longest.last}
else
_ -> {:error, :no_time_window}
end
end
defp find_free_ranges(start_date, end_date, site) do
occupied_ranges =
site
|> Imported.list_all_imports(Imported.SiteImport.completed())
|> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2))
|> Enum.map(&Date.range(&1.start_date, &1.end_date))
Date.range(start_date, end_date)
|> free_ranges(start_date, occupied_ranges, [])
end
# This function recursively finds open ranges that are not yet occupied
# by existing imported data. The idea is that we keep moving a dynamic
# date index `d` from start until the end of `imported_range`, hopping
# over each occupied range, and capturing the open ranges step-by-step
# in the `result` array.
defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do
cond do
Date.diff(occupied_range.last, d) <= 0 ->
free_ranges(import_range, d, rest_of_occupied_ranges, result)
in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 ->
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
true ->
free_range = Date.range(d, occupied_range.first)
result = result ++ [free_range]
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
end
end
defp free_ranges(import_range, d, [], result) do
if Date.diff(import_range.last, d) < 2 do
result
else
result ++ [Date.range(d, import_range.last)]
end
end
defp in_range?(date, range) do
Date.before?(range.first, date) && Date.after?(range.last, date)
end
end end

@ -8,11 +8,9 @@ defmodule Plausible.Imported.Browser do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :browser, :string field :browser, :string
field :browser_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -43,17 +43,9 @@ defmodule Plausible.Imported.CSVImporter do
{table, _, _} = parse_filename!(filename) {table, _, _} = parse_filename!(filename)
s3_structure = input_structure!(table) s3_structure = input_structure!(table)
s3_structure_cols_expr =
s3_structure
|> String.split(",", trim: true)
|> Enum.map_join(", ", fn kv ->
[col, _type] = String.split(kv)
col
end)
statement = statement =
""" """
INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \ INSERT INTO {table:Identifier} \
SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \
FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \ FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \
WHERE date >= {start_date:Date} AND date <= {end_date:Date}\ WHERE date >= {start_date:Date} AND date <= {end_date:Date}\
@ -83,21 +75,20 @@ defmodule Plausible.Imported.CSVImporter do
input_structures = %{ input_structures = %{
"imported_browsers" => "imported_browsers" =>
"date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, browser String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_devices" => "imported_devices" =>
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_entry_pages" => "imported_entry_pages" =>
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32",
"imported_exit_pages" => "imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64",
"date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
"imported_locations" => "imported_locations" =>
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_operating_systems" => "imported_operating_systems" =>
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, operating_system String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_pages" => "imported_pages" =>
"date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64", "date Date, hostname String, page String, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
"imported_sources" => "imported_sources" =>
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" => "imported_visitors" =>
"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64" "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
} }
@ -114,12 +105,12 @@ defmodule Plausible.Imported.CSVImporter do
Date.range(~D[2019-01-01], ~D[2022-01-01]) Date.range(~D[2019-01-01], ~D[2022-01-01])
iex> date_range([]) iex> date_range([])
nil ** (ArgumentError) empty uploads
""" """
@spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil) def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
def date_range([]), do: nil def date_range([]), do: raise(ArgumentError, "empty uploads")
defp date_range([upload | uploads], prev_start_date, prev_end_date) do defp date_range([upload | uploads], prev_start_date, prev_end_date) do
filename = filename =
@ -183,47 +174,4 @@ defmodule Plausible.Imported.CSVImporter do
def parse_filename!(_filename) do def parse_filename!(_filename) do
raise ArgumentError, "invalid filename" raise ArgumentError, "invalid filename"
end end
@doc """
Checks if the provided filename conforms to the expected format.
Examples:
iex> valid_filename?("my_data.csv")
false
iex> valid_filename?("imported_devices_00010101_20250101.csv")
true
"""
@spec valid_filename?(String.t()) :: boolean
def valid_filename?(filename) do
try do
parse_filename!(filename)
else
_ -> true
rescue
_ -> false
end
end
@doc """
Extracts the table name from the provided filename.
Raises if the filename doesn't conform to the expected format.
Examples:
iex> extract_table("my_data.csv")
** (ArgumentError) invalid filename
iex> extract_table("imported_devices_00010101_20250101.csv")
"imported_devices"
"""
@spec extract_table(String.t()) :: String.t()
def extract_table(filename) do
{table, _start_date, _end_date} = parse_filename!(filename)
table
end
end end

@ -11,7 +11,6 @@ defmodule Plausible.Imported.Device do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -11,7 +11,6 @@ defmodule Plausible.Imported.EntryPage do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :entrances, Ch, type: "UInt64" field :entrances, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -8,10 +8,7 @@ defmodule Plausible.Imported.ExitPage do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :exit_page, :string field :exit_page, :string
field :exits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end end
end end

@ -96,28 +96,17 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
end end
defp new_from_report(site_id, import_id, "imported_sources", row) do defp new_from_report(site_id, import_id, "imported_sources", row) do
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
referrer =
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
end
%{ %{
site_id: site_id, site_id: site_id,
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(), source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
referrer: referrer,
# Only `source` exists in GA4 API
utm_source: nil,
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(), utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(), utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(), utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(), utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -131,7 +120,6 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""), hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path), page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently # NOTE: no exits metric in GA4 API currently
exits: 0, exits: 0,
@ -148,7 +136,6 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(), entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
} }
end end
@ -161,10 +148,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
# date: get_date(row), # date: get_date(row),
# exit_page: Map.fetch!(row.dimensions, "exitPage"), # exit_page: Map.fetch!(row.dimensions, "exitPage"),
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(), # exits: row.metrics |> Map.fetch!("sessions") |> parse_number()
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
# } # }
# end # end
@ -182,7 +166,6 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
city: city_data && city_data.id, city: city_data && city_data.id,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -196,7 +179,6 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(), device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -220,11 +202,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser), browser: Map.get(@browser_google_to_plausible, browser, browser),
# Does not exist in GA4 API
browser_version: nil,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -244,10 +223,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os), operating_system: Map.get(@os_google_to_plausible, os, os),
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }

@ -91,7 +91,6 @@ defmodule Plausible.Imported.Importer do
like LiveView, where notifications can be listened for via `handle_info/2`. like LiveView, where notifications can be listened for via `handle_info/2`.
""" """
alias Plausible.Imported
alias Plausible.Imported.SiteImport alias Plausible.Imported.SiteImport
alias Plausible.Repo alias Plausible.Repo
@ -99,7 +98,7 @@ defmodule Plausible.Imported.Importer do
@callback label() :: String.t() @callback label() :: String.t()
@callback email_template() :: String.t() @callback email_template() :: String.t()
@callback parse_args(map()) :: Keyword.t() @callback parse_args(map()) :: Keyword.t()
@callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()} @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()} @callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
@callback on_success(SiteImport.t(), map()) :: :ok @callback on_success(SiteImport.t(), map()) :: :ok
@callback on_failure(SiteImport.t()) :: :ok @callback on_failure(SiteImport.t()) :: :ok
@ -109,7 +108,7 @@ defmodule Plausible.Imported.Importer do
@behaviour Plausible.Imported.Importer @behaviour Plausible.Imported.Importer
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) :: @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress} {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
def new_import(site, user, opts) do def new_import(site, user, opts) do
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1) Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
end end
@ -223,15 +222,11 @@ defmodule Plausible.Imported.Importer do
:ok = listen() :ok = listen()
end end
if not Imported.other_imports_in_progress?(site_import) do opts
opts |> Keyword.put(:import_id, site_import.id)
|> Keyword.put(:import_id, site_import.id) |> Map.new()
|> Map.new() |> Plausible.Workers.ImportAnalytics.new()
|> Plausible.Workers.ImportAnalytics.new() |> Oban.insert()
|> Oban.insert()
else
{:error, :import_in_progress}
end
end end
defp mark_complete(site_import, extra_data, on_success_fun) do defp mark_complete(site_import, extra_data, on_success_fun) do

@ -13,7 +13,6 @@ defmodule Plausible.Imported.Location do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -8,11 +8,9 @@ defmodule Plausible.Imported.OperatingSystem do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :operating_system, :string field :operating_system, :string
field :operating_system_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -9,7 +9,6 @@ defmodule Plausible.Imported.Page do
field :date, :date field :date, :date
field :hostname, :string field :hostname, :string
field :page, :string field :page, :string
field :visits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64" field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"

@ -8,8 +8,6 @@ defmodule Plausible.Imported.Source do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :source, :string field :source, :string
field :referrer, :string
field :utm_source, :string
field :utm_medium, :string field :utm_medium, :string
field :utm_campaign, :string field :utm_campaign, :string
field :utm_content, :string field :utm_content, :string
@ -17,7 +15,6 @@ defmodule Plausible.Imported.Source do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -29,7 +29,6 @@ defmodule Plausible.Ingestion.Event do
| :dc_ip | :dc_ip
| :site_ip_blocklist | :site_ip_blocklist
| :site_country_blocklist | :site_country_blocklist
| :site_page_blocklist
@type t() :: %__MODULE__{ @type t() :: %__MODULE__{
domain: String.t() | nil, domain: String.t() | nil,
@ -44,6 +43,26 @@ defmodule Plausible.Ingestion.Event do
changeset: %Ecto.Changeset{} changeset: %Ecto.Changeset{}
} }
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do def build_and_buffer(%Request{domains: domains} = request) do
processed_events = processed_events =
@ -80,10 +99,6 @@ defmodule Plausible.Ingestion.Event do
[:plausible, :ingest, :event, :dropped] [:plausible, :ingest, :event, :dropped]
end end
def telemetry_pipeline_step_duration() do
[:plausible, :ingest, :pipeline, :step]
end
@spec emit_telemetry_buffered(t()) :: :ok @spec emit_telemetry_buffered(t()) :: :ok
def emit_telemetry_buffered(event) do def emit_telemetry_buffered(event) do
:telemetry.execute(telemetry_event_buffered(), %{}, %{ :telemetry.execute(telemetry_event_buffered(), %{}, %{
@ -103,42 +118,33 @@ defmodule Plausible.Ingestion.Event do
defp pipeline() do defp pipeline() do
[ [
drop_datacenter_ip: &drop_datacenter_ip/1, &drop_datacenter_ip/1,
drop_shield_rule_page: &drop_shield_rule_page/1, &drop_shield_rule_ip/1,
drop_shield_rule_ip: &drop_shield_rule_ip/1, &put_geolocation/1,
put_geolocation: &put_geolocation/1, &drop_shield_rule_country/1,
drop_shield_rule_country: &drop_shield_rule_country/1, &put_user_agent/1,
put_user_agent: &put_user_agent/1, &put_basic_info/1,
put_basic_info: &put_basic_info/1, &put_referrer/1,
put_referrer: &put_referrer/1, &put_utm_tags/1,
put_utm_tags: &put_utm_tags/1, &put_props/1,
put_props: &put_props/1, &put_revenue/1,
put_revenue: &put_revenue/1, &put_salts/1,
put_salts: &put_salts/1, &put_user_id/1,
put_user_id: &put_user_id/1, &validate_clickhouse_event/1,
validate_clickhouse_event: &validate_clickhouse_event/1, &register_session/1,
register_session: &register_session/1, &write_to_buffer/1
write_to_buffer: &write_to_buffer/1
] ]
end end
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event -> Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( case pipeline_step.(acc_event) do
telemetry_pipeline_step_duration(), %__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
fn -> execute_step(step_fn, acc_event) end, %__MODULE__{dropped?: false} = event -> {:cont, event}
%{step: "#{step_name}"} end
)
end) end)
end end
defp execute_step(step_fn, acc_event) do
case step_fn.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end
defp new(domain, request) do defp new(domain, request) do
struct!(__MODULE__, domain: domain, request: request) struct!(__MODULE__, domain: domain, request: request)
end end
@ -176,18 +182,15 @@ defmodule Plausible.Ingestion.Event do
end end
defp drop_shield_rule_ip(%__MODULE__{} = event) do defp drop_shield_rule_ip(%__MODULE__{} = event) do
if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do domain = event.domain
drop(event, :site_ip_blocklist) address = event.request.remote_ip
else
event
end
end
defp drop_shield_rule_page(%__MODULE__{} = event) do case Plausible.Shield.IPRuleCache.get({domain, address}) do
if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do %Plausible.Shield.IPRule{action: :deny} ->
drop(event, :site_page_blocklist) drop(event, :site_ip_blocklist)
else
event _ ->
event
end end
end end
@ -260,10 +263,12 @@ defmodule Plausible.Ingestion.Event do
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
) )
when is_binary(domain) and is_binary(cc) do when is_binary(domain) and is_binary(cc) do
if Plausible.Shields.country_blocked?(domain, cc) do case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
drop(event, :site_country_blocklist) %Plausible.Shield.CountryRule{action: :deny} ->
else drop(event, :site_country_blocklist)
event
_ ->
event
end end
end end
@ -339,7 +344,8 @@ defmodule Plausible.Ingestion.Event do
%{ %{
event event
| clickhouse_event: ClickhouseEventV2.merge_session(event.clickhouse_event, session) | clickhouse_event:
Map.merge(event.clickhouse_event, Map.take(session, @session_properties))
} }
end end

@ -47,14 +47,7 @@ defmodule Plausible.Session.CacheStore do
do: event.pathname, do: event.pathname,
else: session.entry_page else: session.entry_page
), ),
hostname:
if(event.name == "pageview" and session.hostname == "",
do: event.hostname,
else: session.hostname
),
exit_page: if(event.name == "pageview", do: event.pathname, else: session.exit_page), exit_page: if(event.name == "pageview", do: event.pathname, else: session.exit_page),
exit_page_hostname:
if(event.name == "pageview", do: event.hostname, else: session.exit_page_hostname),
is_bounce: false, is_bounce: false,
duration: Timex.diff(event.timestamp, session.start, :second) |> abs, duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews: pageviews:
@ -67,12 +60,11 @@ defmodule Plausible.Session.CacheStore do
%Plausible.ClickhouseSessionV2{ %Plausible.ClickhouseSessionV2{
sign: 1, sign: 1,
session_id: Plausible.ClickhouseSessionV2.random_uint64(), session_id: Plausible.ClickhouseSessionV2.random_uint64(),
hostname: if(event.name == "pageview", do: event.hostname, else: ""), hostname: event.hostname,
site_id: event.site_id, site_id: event.site_id,
user_id: event.user_id, user_id: event.user_id,
entry_page: if(event.name == "pageview", do: event.pathname, else: ""), entry_page: if(event.name == "pageview", do: event.pathname, else: ""),
exit_page: if(event.name == "pageview", do: event.pathname, else: ""), exit_page: if(event.name == "pageview", do: event.pathname, else: ""),
exit_page_hostname: if(event.name == "pageview", do: event.hostname, else: ""),
is_bounce: true, is_bounce: true,
duration: 0, duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0), pageviews: if(event.name == "pageview", do: 1, else: 0),

@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do
|> cast(attrs, [:site_id, :country_code]) |> cast(attrs, [:site_id, :country_code])
|> validate_required([:site_id, :country_code]) |> validate_required([:site_id, :country_code])
|> validate_length(:country_code, is: 2) |> validate_length(:country_code, is: 2)
|> validate_change(:country_code, fn :country_code, cc -> |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc ->
if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do
[] []
else else

@ -1,85 +0,0 @@
defmodule CompiledRegex do
@moduledoc """
Ensures that the regex is compiled on load
"""
use Ecto.Type
def type, do: :string
def cast(val) when is_binary(val), do: {:ok, val}
def cast(_), do: :error
def load(val), do: {:ok, Regex.compile!(val)}
def dump(val), do: {:ok, val}
end
defmodule Plausible.Shield.PageRule do
@moduledoc """
Schema for Pages block list
"""
use Ecto.Schema
import Ecto.Changeset
@type t() :: %__MODULE__{}
@primary_key {:id, :binary_id, autogenerate: true}
schema "shield_rules_page" do
belongs_to :site, Plausible.Site
field :page_path, :string
field :page_path_pattern, CompiledRegex
field :action, Ecto.Enum, values: [:deny, :allow], default: :deny
field :added_by, :string
# If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache`
field :from_cache?, :boolean, virtual: true, default: false
timestamps()
end
def changeset(rule \\ %__MODULE__{}, attrs) do
rule
|> cast(attrs, [:site_id, :page_path])
|> validate_required([:site_id, :page_path])
|> validate_length(:page_path, max: 250)
|> validate_change(:page_path, fn :page_path, p ->
if not String.starts_with?(p, "/") do
[page_path: "must start with /"]
else
[]
end
end)
|> store_regex()
|> unique_constraint(:page_path_pattern,
name: :shield_rules_page_site_id_page_path_pattern_index,
error_key: :page_path,
message: "rule already exists"
)
end
defp store_regex(changeset) do
case get_field(changeset, :page_path) do
"/" <> _ = page_path ->
regex =
page_path
|> Regex.escape()
|> String.replace("\\*\\*", ".*")
|> String.replace("\\*", ".*")
regex = "^#{regex}$"
verify_valid_regex(changeset, regex)
_ ->
changeset
end
end
defp verify_valid_regex(changeset, regex) do
case Regex.compile(regex) do
{:ok, _} ->
put_change(changeset, :page_path_pattern, regex)
{:error, _} ->
add_error(changeset, :page_path, "could not compile regular expression")
end
end
end

@ -1,65 +0,0 @@
defmodule Plausible.Shield.PageRuleCache do
@moduledoc """
Allows retrieving Page Rules by domain
"""
alias Plausible.Shield.PageRule
import Ecto.Query
use Plausible.Cache
@cache_name :page_blocklist_by_domain
@cached_schema_fields ~w(
id
page_path_pattern
action
)a
@impl true
def name(), do: @cache_name
@impl true
def child_id(), do: :cache_page_blocklist
@impl true
def count_all() do
Plausible.Repo.aggregate(PageRule, :count)
end
@impl true
def base_db_query() do
from rule in PageRule,
inner_join: s in assoc(rule, :site),
select: {
s.domain,
s.domain_changed_from,
%{struct(rule, ^@cached_schema_fields) | from_cache?: true}
}
end
@impl true
def get_from_source(domain) do
query =
base_db_query()
|> where([..., site], site.domain == ^domain)
case Plausible.Repo.one(query) do
{_, _, rule} -> %PageRule{rule | from_cache?: false}
_any -> nil
end
end
@impl true
def unwrap_cache_keys(items) do
Enum.reduce(items, [], fn
{domain, nil, object}, acc ->
[{domain, object} | acc]
{domain, domain_changed_from, object}, acc ->
[
{domain, object},
{domain_changed_from, object} | acc
]
end)
end
end

@ -13,63 +13,11 @@ defmodule Plausible.Shields do
@maximum_country_rules 30 @maximum_country_rules 30
def maximum_country_rules(), do: @maximum_country_rules def maximum_country_rules(), do: @maximum_country_rules
@maximum_page_rules 30
def maximum_page_rules(), do: @maximum_page_rules
@spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()]
def list_ip_rules(site_or_id) do def list_ip_rules(site_or_id) do
list(Shield.IPRule, site_or_id) list(Shield.IPRule, site_or_id)
end end
@spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def ip_blocked?(%Site{domain: domain}, address) do
ip_blocked?(domain, address)
end
def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do
case Shield.IPRuleCache.get({domain, address}) do
%Shield.IPRule{action: :deny} ->
true
_ ->
false
end
end
@spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def page_blocked?(%Site{domain: domain}, address) do
page_blocked?(domain, address)
end
def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do
page_rules = Shield.PageRuleCache.get(domain)
if page_rules do
page_rules
|> List.wrap()
|> Enum.find_value(false, fn rule ->
rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname)
end)
else
false
end
end
@spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def country_blocked?(%Site{domain: domain}, country_code) do
country_blocked?(domain, country_code)
end
def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do
case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do
%Shield.CountryRule{action: :deny} ->
true
_ ->
false
end
end
@spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()}
def add_ip_rule(site_or_id, params, opts \\ []) do def add_ip_rule(site_or_id, params, opts \\ []) do
@ -111,28 +59,6 @@ defmodule Plausible.Shields do
count(Shield.CountryRule, site_or_id) count(Shield.CountryRule, site_or_id)
end end
@spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()]
def list_page_rules(site_or_id) do
list(Shield.PageRule, site_or_id)
end
@spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()}
def add_page_rule(site_or_id, params, opts \\ []) do
opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules})
add(Shield.PageRule, site_or_id, params, opts)
end
@spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok
def remove_page_rule(site_or_id, rule_id) do
remove(Shield.PageRule, site_or_id, rule_id)
end
@spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer()
def count_page_rules(site_or_id) do
count(Shield.PageRule, site_or_id)
end
defp list(schema, %Site{id: id}) do defp list(schema, %Site{id: id}) do
list(schema, id) list(schema, id)
end end

@ -1,136 +0,0 @@
defmodule Plausible.Shields.CountryTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_country_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{})
assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported country", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"})
assert changeset.errors == [country_code: {"is invalid", []}]
refute changeset.valid?
end
test "incorrect country format", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"})
assert changeset.errors ==
[
{:country_code, {"is invalid", []}},
{:country_code,
{"should be %{count} character(s)",
[count: 2, validation: :length, kind: :is, type: :string]}}
]
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"})
refute changeset.valid?
assert changeset.errors == [
country_code:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_country_site_id_country_code_index"}
]}
]
end
test "over limit", %{site: site} do
country_codes =
Location.Country.all()
|> Enum.take(Plausible.Shields.maximum_country_rules())
|> Enum.map(& &1.alpha_2)
for cc <- country_codes do
assert {:ok, _} =
add_country_rule(site, %{"country_code" => cc})
end
assert count_country_rules(site) == maximum_country_rules()
assert {:error, changeset} =
add_country_rule(site, %{"country_code" => "US"})
refute changeset.valid?
assert changeset.errors == [country_code: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_country_rule(site, %{"country_code" => "EE"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_country_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_country_rule(site, %{"country_code" => "EE"})
assert remove_country_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.CountryRule, rule.id)
assert remove_country_rule(site, rule.id) == :ok
end
end
describe "list_country_rules/1" do
test "empty", %{site: site} do
assert(list_country_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_country_rule(site, %{"country_code" => "EE"})
:timer.sleep(1000)
{:ok, r2} = add_country_rule(site, %{"country_code" => "PL"})
assert [^r2, ^r1] = list_country_rules(site)
end
end
describe "count_country_rules/1" do
test "counts", %{site: site} do
assert count_country_rules(site) == 0
{:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert count_country_rules(site) == 1
{:ok, _} = add_country_rule(site, %{"country_code" => "PL"})
assert count_country_rules(site) == 2
end
end
describe "Country Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_country_rules(site.id) == 0
assert list_country_rules(site.id) == []
assert {:ok, rule} =
add_country_rule(site.id, %{"country_code" => "EE"})
add_country_rule(site2, %{"country_code" => "PL"})
assert count_country_rules(site) == 1
assert [^rule] = list_country_rules(site)
assert rule.country_code == "EE"
assert rule.action == :deny
refute rule.from_cache?
assert country_blocked?(site, "ee")
refute country_blocked?(site, "xx")
end
end
end

@ -1,147 +0,0 @@
defmodule Plausible.Shields.IPTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_ip_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{})
assert changeset.errors == [inet: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported netmask", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"})
assert changeset.errors == [inet: {"netmask unsupported", []}]
refute changeset.valid?
assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"})
end
test "incorrect ip", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"})
assert changeset.errors == [
inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]}
]
refute changeset.valid?
end
test "non-strict IPs", %{site: site} do
assert {:error, _} = add_ip_rule(site, %{"inet" => "111"})
end
test "double insert", %{site: site} do
assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
refute changeset.valid?
assert changeset.errors == [
inet:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_ip_site_id_inet_index"}
]}
]
end
test "ipv6", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "ipv4", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "over limit", %{site: site} do
for i <- 1..maximum_ip_rules() do
assert {:ok, _} =
add_ip_rule(site, %{"inet" => "1.1.1.#{i}"})
end
assert count_ip_rules(site) == maximum_ip_rules()
assert {:error, changeset} =
add_ip_rule(site, %{"inet" => "1.1.1.31"})
refute changeset.valid?
assert changeset.errors == [inet: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_ip_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert remove_ip_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.IPRule, rule.id)
assert remove_ip_rule(site, rule.id) == :ok
end
end
describe "list_ip_rules/1" do
test "empty", %{site: site} do
assert(list_ip_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
:timer.sleep(1000)
{:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert [^r2, ^r1] = list_ip_rules(site)
end
end
describe "count_ip_rules/1" do
test "counts", %{site: site} do
assert count_ip_rules(site) == 0
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert count_ip_rules(site) == 1
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert count_ip_rules(site) == 2
end
end
describe "IP Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_ip_rules(site.id) == 0
assert list_ip_rules(site.id) == []
assert {:ok, rule} =
add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"})
add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"})
assert count_ip_rules(site) == 1
assert [^rule] = list_ip_rules(site)
assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32}
assert rule.description == "Localhost"
assert rule.action == :deny
refute rule.from_cache?
assert ip_blocked?(site, "127.0.0.1")
refute ip_blocked?(site, "127.0.0.2")
end
end
end

@ -1,174 +0,0 @@
defmodule Plausible.Shields.PageTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_page_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{})
assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "no slash", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"})
assert changeset.errors == [page_path: {"must start with /", []}]
refute changeset.valid?
end
test "lengthy", %{site: site} do
long = "/" <> :binary.copy("a", 251)
assert {:error, changeset} = add_page_rule(site, %{"page_path" => long})
assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"})
refute changeset.valid?
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "equivalent rules are counted as dupes", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"})
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "regex storage: wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"})
assert rule.page_path_pattern == "^/test/.*$"
end
test "regex storage: no wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert rule.page_path_pattern == "^/test$"
end
test "regex storage: escaping", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"})
assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$"
end
test "over limit", %{site: site} do
for i <- 1..maximum_page_rules() do
assert {:ok, _} =
add_page_rule(site, %{"page_path" => "/test/#{i}"})
end
assert count_page_rules(site) == maximum_page_rules()
assert {:error, changeset} =
add_page_rule(site, %{"page_path" => "/test/31"})
refute changeset.valid?
assert changeset.errors == [page_path: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_page_rule(site, %{"page_path" => "/test"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "page pattern matching" do
test "no wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
end
test "wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
refute page_blocked?(site, "/test")
assert page_blocked?(site, "/test/")
assert page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
refute page_blocked?(site, "/testing")
end
end
describe "remove_page_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert remove_page_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.PageRule, rule.id)
assert remove_page_rule(site, rule.id) == :ok
end
end
describe "list_page_rules/1" do
test "empty", %{site: site} do
assert(list_page_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_page_rule(site, %{"page_path" => "/test"})
:timer.sleep(1000)
{:ok, r2} = add_page_rule(site, %{"page_path" => "/test"})
assert [^r2, ^r1] = list_page_rules(site)
end
end
describe "count_page_rules/1" do
test "counts", %{site: site} do
assert count_page_rules(site) == 0
{:ok, _} = add_page_rule(site, %{"page_path" => "/test1"})
assert count_page_rules(site) == 1
{:ok, _} = add_page_rule(site, %{"page_path" => "/test2"})
assert count_page_rules(site) == 2
end
end
describe "Page Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_page_rules(site.id) == 0
assert list_page_rules(site.id) == []
assert {:ok, rule} =
add_page_rule(site.id, %{
"page_path" => "/test"
})
add_page_rule(site2, %{"page_path" => "/test"})
assert count_page_rules(site) == 1
assert [%{id: rule_id}] = list_page_rules(site)
assert rule.id == rule_id
assert rule.page_path == "/test"
assert rule.action == :deny
refute rule.from_cache?
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/testing")
end
end
end

@ -59,7 +59,7 @@ defmodule Plausible.Site do
# `site` in `assigns`. # `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true field :earliest_import_start_date, :date, virtual: true
field :latest_import_end_date, :date, virtual: true field :earliest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps() timestamps()

@ -5,6 +5,19 @@ defmodule Plausible.Stats.Aggregate do
import Ecto.Query import Ecto.Query
alias Plausible.Stats.{Query, Util} alias Plausible.Stats.{Query, Util}
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [
:visitors,
:pageviews,
:events,
:sample_percent,
:conversion_rate,
:total_visitors
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
def aggregate(site, query, metrics) do def aggregate(site, query, metrics) do
{currency, metrics} = {currency, metrics} =
on_full_build do on_full_build do
@ -15,17 +28,18 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics) Query.trace(query, metrics)
{event_metrics, session_metrics, other_metrics} = event_metrics =
metrics metrics
|> Util.maybe_add_visitors_metric() |> Util.maybe_add_visitors_metric()
|> Plausible.Stats.TableDecider.partition_metrics(query) |> Enum.filter(&(&1 in @event_metrics))
event_task = fn -> aggregate_events(site, query, event_metrics) end event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
session_task = fn -> aggregate_sessions(site, query, session_metrics) end session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task = time_on_page_task =
if :time_on_page in other_metrics do if :time_on_page in metrics do
fn -> aggregate_time_on_page(site, query) end fn -> aggregate_time_on_page(site, query) end
else else
fn -> %{} end fn -> %{} end

@ -3,7 +3,7 @@ defmodule Plausible.Stats.Base do
use Plausible use Plausible
use Plausible.Stats.Fragments use Plausible.Stats.Fragments
alias Plausible.Stats.{Query, Filters, TableDecider} alias Plausible.Stats.{Query, Filters}
alias Plausible.Timezones alias Plausible.Timezones
import Ecto.Query import Ecto.Query
@ -15,7 +15,7 @@ defmodule Plausible.Stats.Base do
def base_event_query(site, query) do def base_event_query(site, query) do
events_q = query_events(site, query) events_q = query_events(site, query)
if TableDecider.events_join_sessions?(query) do if Enum.any?(Filters.visit_props(), &query.filters["visit:" <> &1]) do
sessions_q = sessions_q =
from( from(
s in query_sessions(site, query), s in query_sessions(site, query),
@ -30,15 +30,11 @@ defmodule Plausible.Stats.Base do
on: e.session_id == sq.session_id on: e.session_id == sq.session_id
) )
else else
if query.experimental_reduced_joins? do events_q
events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)
else
events_q
end
end end
end end
defp query_events(site, query) do def query_events(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site) {first_datetime, last_datetime} = utc_boundaries(query, site)
q = q =
@ -52,10 +48,7 @@ defmodule Plausible.Stats.Base do
q = Plausible.Stats.Sampling.add_query_hint(q, query) q = Plausible.Stats.Sampling.add_query_hint(q, query)
end end
q = q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname))
q
|> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))
|> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))
q = q =
case query.filters["event:name"] do case query.filters["event:name"] do
@ -129,6 +122,17 @@ defmodule Plausible.Stats.Base do
q q
end end
@api_prop_name_to_db %{
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id"
}
def query_sessions(site, query) do def query_sessions(site, query) do
{first_datetime, last_datetime} = {first_datetime, last_datetime} =
utc_boundaries(query, site) utc_boundaries(query, site)
@ -136,7 +140,8 @@ defmodule Plausible.Stats.Base do
q = from(s in "sessions_v2", where: s.site_id == ^site.id) q = from(s in "sessions_v2", where: s.site_id == ^site.id)
sessions_q = sessions_q =
if query.experimental_session_count? do if FunWithFlags.enabled?(:experimental_session_count, for: site) or
query.experimental_session_count? do
from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
else else
from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime
@ -146,24 +151,9 @@ defmodule Plausible.Stats.Base do
sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query) sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
end end
filter_by_entry_props(sessions_q, query) sessions_q = filter_by_entry_props(sessions_q, query)
|> filter_by_visit_props(Filters.visit_props(), query)
end
@api_prop_name_to_db %{ Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id",
"entry_page_hostname" => "hostname"
}
defp filter_by_visit_props(q, visit_props, query) do
Enum.reduce(visit_props, q, fn prop_name, sessions_q ->
filter_key = "visit:" <> prop_name filter_key = "visit:" <> prop_name
db_field = db_field =
@ -247,13 +237,6 @@ defmodule Plausible.Stats.Base do
} }
end end
defp select_event_metric(:visits) do
%{
visits:
dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))
}
end
on_full_build do on_full_build do
defp select_event_metric(:total_revenue) do defp select_event_metric(:total_revenue) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()} %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}

@ -6,7 +6,7 @@ defmodule Plausible.Stats.Breakdown do
import Plausible.Stats.{Base, Imported} import Plausible.Stats.{Base, Imported}
import Ecto.Query import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util, TableDecider} alias Plausible.Stats.{Query, Util}
@no_ref "Direct / None" @no_ref "Direct / None"
@not_set "(not set)" @not_set "(not set)"
@ -103,6 +103,7 @@ defmodule Plausible.Stats.Breakdown do
{event_q, page_q} -> {event_q, page_q} ->
from( from(
e in subquery(union_all(event_q, ^page_q)), e in subquery(union_all(event_q, ^page_q)),
# :TODO: Handle other orderings
order_by: [desc: e.visitors] order_by: [desc: e.visitors]
) )
|> apply_pagination(pagination) |> apply_pagination(pagination)
@ -199,35 +200,9 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
case breakdown_table(query, metrics, property) do breakdown_sessions(site, query, property, metrics_to_select)
:session -> |> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
breakdown_sessions(site, query, property, metrics_to_select) |> paginate_and_execute(metrics, pagination)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
:event ->
breakdown_events(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end
end
# Backwards compatibility
# defp breakdown_table(%Query{experimental_reduced_joins?: false}, _, _), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page_hostname"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page_hostname"), do: :session
defp breakdown_table(query, metrics, property) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, property)
if not Enum.empty?(session_metrics) do
:session
else
:event
end
end end
defp zip_results(event_result, session_result, property, metrics) do defp zip_results(event_result, session_result, property, metrics) do
@ -298,6 +273,94 @@ defmodule Plausible.Stats.Breakdown do
end end
defp breakdown_time_on_page(site, query, pages) do defp breakdown_time_on_page(site, query, pages) do
if FunWithFlags.enabled?(:window_time_on_page) do
window_breakdown_time_on_page(site, query, pages)
else
neighbor_breakdown_time_on_page(site, query, pages)
end
end
defp neighbor_breakdown_time_on_page(site, query, pages) do
q =
from(
e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
select: {
fragment("? as p", e.pathname),
fragment("? as t", e.timestamp),
fragment("? as s", e.session_id)
},
order_by: [e.session_id, e.timestamp]
)
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
select =
if query.include_imported do
"sum(td), count(case when p2 != p then 1 end)"
else
"round(sum(td)/count(case when p2 != p then 1 end))"
end
pages_idx = length(base_query_raw_params)
params = base_query_raw_params ++ [pages]
time_query = "
SELECT
p,
#{select}
FROM
(SELECT
p,
p2,
sum(t2-t) as td
FROM
(SELECT
*,
neighbor(t, 1) as t2,
neighbor(p, 1) as p2,
neighbor(s, 1) as s2
FROM (#{base_query_raw}))
WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}
GROUP BY p,p2,s)
GROUP BY p"
{:ok, res} = ClickhouseRepo.query(time_query, params)
if query.include_imported do
# Imported page views have pre-calculated values
res =
res.rows
|> Enum.map(fn [page, time, visits] -> {page, {time, visits}} end)
|> Enum.into(%{})
from(
i in "imported_pages",
group_by: i.page,
where: i.site_id == ^site.id,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.page in ^pages,
select: %{
page: i.page,
pageviews: fragment("sum(?) - sum(?)", i.pageviews, i.exits),
time_on_page: sum(i.time_on_page)
}
)
|> ClickhouseRepo.all()
|> Enum.reduce(res, fn %{page: page, pageviews: pageviews, time_on_page: time}, res ->
{restime, resviews} = Map.get(res, page, {0, 0})
Map.put(res, page, {restime + time, resviews + pageviews})
end)
|> Enum.map(fn
{page, {_, 0}} -> {page, nil}
{page, {time, pageviews}} -> {page, time / pageviews}
end)
|> Enum.into(%{})
else
res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{})
end
end
defp window_breakdown_time_on_page(site, query, pages) do
import Ecto.Query import Ecto.Query
windowed_pages_q = windowed_pages_q =
@ -466,8 +529,6 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:entry_page") do defp do_group_by(q, "visit:entry_page") do
from( from(
s in q, s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.entry_page, group_by: s.entry_page,
select_merge: %{entry_page: s.entry_page}, select_merge: %{entry_page: s.entry_page},
order_by: {:asc, s.entry_page} order_by: {:asc, s.entry_page}
@ -477,8 +538,6 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:exit_page") do defp do_group_by(q, "visit:exit_page") do
from( from(
s in q, s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.exit_page, group_by: s.exit_page,
select_merge: %{exit_page: s.exit_page}, select_merge: %{exit_page: s.exit_page},
order_by: {:asc, s.exit_page} order_by: {:asc, s.exit_page}

@ -208,12 +208,11 @@ defmodule Plausible.Stats.FilterSuggestions do
"operating_system" -> :operating_system "operating_system" -> :operating_system
"operating_system_version" -> :operating_system_version "operating_system_version" -> :operating_system_version
"screen_size" -> :screen_size "screen_size" -> :screen_size
"hostname" -> :hostname
_ -> :unknown _ -> :unknown
end end
q = q =
if(filter_name == :pathname or filter_name == :hostname, if(filter_name == :pathname,
do: base_event_query(site, query), do: base_event_query(site, query),
else: query_sessions(site, query) else: query_sessions(site, query)
) )
@ -231,12 +230,6 @@ defmodule Plausible.Stats.FilterSuggestions do
where: fragment("? ilike ?", e.pathname, ^filter_query) where: fragment("? ilike ?", e.pathname, ^filter_query)
) )
:hostname ->
from(e in q,
select: e.hostname,
where: fragment("? ilike ?", e.hostname, ^filter_query)
)
:entry_page -> :entry_page ->
from(e in q, from(e in q,
select: e.entry_page, select: e.entry_page,

@ -31,7 +31,7 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do
{is_negated, val} = parse_negated_prefix(val) {is_negated, val} = parse_negated_prefix(val)
{is_contains, val} = parse_contains_prefix(val) {is_contains, val} = parse_contains_prefix(val)
is_list = list_expression?(val) is_list = list_expression?(val)
is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val) is_wildcard = String.contains?(key, ["page", "goal"]) && wildcard_expression?(val)
val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val) val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
val = if key == "goal", do: wrap_goal_value(val), else: val val = if key == "goal", do: wrap_goal_value(val), else: val

@ -23,22 +23,11 @@ defmodule Plausible.Stats.Filters do
:region, :region,
:city, :city,
:entry_page, :entry_page,
:exit_page, :exit_page
:entry_page_hostname,
:exit_page_hostname
] ]
def visit_props(), do: @visit_props |> Enum.map(&to_string/1) def visit_props(), do: @visit_props |> Enum.map(&to_string/1)
@event_table_visit_props @visit_props -- @event_props [:name, :page, :goal]
[
:entry_page,
:exit_page,
:entry_page_hostname,
:exit_page_hostname
]
def event_table_visit_props(), do: @event_table_visit_props |> Enum.map(&to_string/1)
@event_props [:name, :page, :goal, :hostname]
def event_props(), do: @event_props |> Enum.map(&to_string/1) def event_props(), do: @event_props |> Enum.map(&to_string/1)

@ -9,8 +9,7 @@ defmodule Plausible.Stats.Query do
imported_data_requested: false, imported_data_requested: false,
include_imported: false, include_imported: false,
now: nil, now: nil,
experimental_session_count?: false, experimental_session_count?: false
experimental_reduced_joins?: false
require OpenTelemetry.Tracer, as: Tracer require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Interval} alias Plausible.Stats.{Filters, Interval}
@ -23,8 +22,7 @@ defmodule Plausible.Stats.Query do
query = query =
__MODULE__ __MODULE__
|> struct!(now: now) |> struct!(now: now)
|> put_experimental_session_count(site, params) |> put_experimental_session_count(params)
|> put_experimental_reduced_joins(site, params)
|> put_period(site, params) |> put_period(site, params)
|> put_interval(params) |> put_interval(params)
|> put_parsed_filters(params) |> put_parsed_filters(params)
@ -38,27 +36,11 @@ defmodule Plausible.Stats.Query do
query query
end end
defp put_experimental_session_count(query, site, params) do defp put_experimental_session_count(query, params) do
if Map.has_key?(params, "experimental_session_count") do if Map.get(params, "experimental_session_count") == "true" do
struct!(query, struct!(query, experimental_session_count?: true)
experimental_session_count?: Map.get(params, "experimental_session_count") == "true"
)
else else
struct!(query, query
experimental_session_count?: FunWithFlags.enabled?(:experimental_session_count, for: site)
)
end
end
defp put_experimental_reduced_joins(query, site, params) do
if Map.has_key?(params, "experimental_reduced_joins") do
struct!(query,
experimental_reduced_joins?: Map.get(params, "experimental_reduced_joins") == "true"
)
else
struct!(query,
experimental_reduced_joins?: FunWithFlags.enabled?(:experimental_reduced_joins, for: site)
)
end end
end end
@ -279,8 +261,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean() @spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do def include_imported?(query, site, requested?) do
cond do cond do
is_nil(site.latest_import_end_date) -> false is_nil(site.earliest_import_end_date) -> false
Date.after?(query.date_range.first, site.latest_import_end_date) -> false Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
Enum.any?(query.filters) -> false Enum.any?(query.filters) -> false
query.period == "realtime" -> false query.period == "realtime" -> false
true -> requested? true -> requested?

@ -1,111 +0,0 @@
defmodule Plausible.Stats.TableDecider do
@moduledoc """
This module contains logic for deciding which tables need to be queried given a query
and metrics, with the purpose of reducing the number of queries and JOINs needed to perform.
"""
import Enum, only: [empty?: 1]
alias Plausible.Stats.Query
def events_join_sessions?(query) do
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
end
def partition_metrics(metrics, query, breakdown_property \\ nil) do
%{
event: event_only_metrics,
session: session_only_metrics,
either: either_metrics,
other: other_metrics,
sample_percent: sample_percent
} =
partition(metrics, query, &metric_partitioner/2)
# Treat breakdown property as yet another filter
filters =
if breakdown_property do
Map.put(query.filters, breakdown_property, nil)
else
query.filters
end
%{event: event_only_filters, session: session_only_filters} =
partition(filters, query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
empty?(event_only_metrics) && empty?(event_only_filters) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) && empty?(session_only_filters) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Filters on both events and sessions, but only one kind of metric
empty?(event_only_metrics) ->
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
empty?(session_only_metrics) ->
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
# Default: prefer sessions
true ->
{event_only_metrics ++ sample_percent,
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
end
end
defp metric_partitioner(_, :conversion_rate), do: :event
defp metric_partitioner(_, :average_revenue), do: :event
defp metric_partitioner(_, :total_revenue), do: :event
defp metric_partitioner(_, :pageviews), do: :event
defp metric_partitioner(_, :events), do: :event
defp metric_partitioner(_, :bounce_rate), do: :session
defp metric_partitioner(_, :visit_duration), do: :session
defp metric_partitioner(_, :views_per_visit), do: :session
# Metrics which used to only be queried from one table but can be calculated from either
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visits), do: :either
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visitors), do: :either
defp metric_partitioner(_, :visits), do: :session
defp metric_partitioner(_, :visitors), do: :event
# Calculated metrics - handled on callsite separately from other metrics.
defp metric_partitioner(_, :time_on_page), do: :other
defp metric_partitioner(_, :total_visitors), do: :other
defp metric_partitioner(_, :percentage), do: :other
# Sample percentage is included in both tables if queried.
defp metric_partitioner(_, :sample_percent), do: :sample_percent
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
end
defp metric_partitioner(_, _), do: :either
defp filters_partitioner(_, {"event:" <> _, _}), do: :event
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
do: :either
defp filters_partitioner(_, {"visit:" <> _, _}),
do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
end
defp filters_partitioner(_, _), do: :either
@default %{event: [], session: [], either: [], other: [], sample_percent: []}
defp partition(values, query, partitioner) do
Enum.reduce(values, @default, fn value, acc ->
key = partitioner.(query, value)
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
end)
end
end

@ -18,11 +18,15 @@ defmodule Plausible.Stats.Timeseries do
@typep value :: nil | integer() | float() @typep value :: nil | integer() | float()
@type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()}) @type results :: nonempty_list(%{required(:date) => Date.t(), required(metric()) => value()})
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [:visitors, :pageviews, :events, :conversion_rate] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit]
def timeseries(site, query, metrics) do def timeseries(site, query, metrics) do
steps = buckets(query) steps = buckets(query)
{event_metrics, session_metrics, _} = event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
Plausible.Stats.TableDecider.partition_metrics(metrics, query) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
{currency, event_metrics} = {currency, event_metrics} =
on_full_build do on_full_build do

@ -47,16 +47,6 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
unit: {:native, :millisecond}, unit: {:native, :millisecond},
measurement: :duration measurement: :duration
), ),
distribution(
metric_prefix ++ [:ingest, :events, :pipeline, :steps],
event_name: Ingestion.Event.telemetry_pipeline_step_duration(),
reporter_options: [
buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000]
],
unit: {:native, :microsecond},
measurement: :duration,
tags: [:step]
),
counter( counter(
metric_prefix ++ [:ingest, :events, :buffered, :total], metric_prefix ++ [:ingest, :events, :buffered, :total],
event_name: Ingestion.Event.telemetry_event_buffered() event_name: Ingestion.Event.telemetry_event_buffered()
@ -120,19 +110,6 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
}) })
end end
def measure_duration(event, fun, meta \\ %{}) when is_function(fun, 0) do
{duration, result} = time_it(fun)
:telemetry.execute(event, %{duration: duration}, meta)
result
end
defp time_it(fun) do
start = System.monotonic_time()
result = fun.()
stop = System.monotonic_time()
{stop - start, result}
end
defp write_buffer_metrics(metric_prefix, poll_rate) do defp write_buffer_metrics(metric_prefix, poll_rate) do
Polling.build( Polling.build(
:write_buffer_metrics, :write_buffer_metrics,

@ -1,50 +0,0 @@
defmodule PlausibleWeb.Components.FirstDashboardLaunchBanner do
@moduledoc """
A banner that appears on the first dashboard launch
"""
use Phoenix.Component
use Phoenix.HTML
attr(:site, Plausible.Site, required: true)
def set(assigns) do
~H"""
<script>
sessionStorage.setItem('<%= storage_key(@site) %>', false);
</script>
"""
end
attr(:site, Plausible.Site, required: true)
def render(assigns) do
~H"""
<div
x-cloak
x-data={x_data(@site)}
class="w-full px-4 text-sm font-bold text-center text-blue-900 bg-blue-200 rounded transition"
style="top: 91px"
role="alert"
x-bind:class="! show ? 'hidden' : ''"
x-init={x_init(@site)}
>
<%= link("Team members, email reports and GA import. Explore more →",
to: "/#{URI.encode_www_form(@site.domain)}/settings/email-reports",
class: "py-2 block"
) %>
</div>
"""
end
defp x_data(site) do
"{show: !!sessionStorage.getItem('#{storage_key(site)}')}"
end
defp x_init(site) do
"setTimeout(() => sessionStorage.removeItem('#{storage_key(site)}'), 3000)"
end
defp storage_key(site) do
"dashboard_seen_#{site.domain}"
end
end

@ -4,6 +4,7 @@ defmodule PlausibleWeb.Api.StatsController do
use Plausible.Repo use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Imported.SiteImport
alias Plausible.Stats alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons} alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H alias PlausibleWeb.Api.Helpers, as: H
@ -67,8 +68,8 @@ defmodule PlausibleWeb.Api.StatsController do
* `with_imported` - boolean indicating whether the Google Analytics data * `with_imported` - boolean indicating whether the Google Analytics data
was queried or not. was queried or not.
* `imports_exist` - boolean indicating whether there are any completed * `imported_source` - the source of the imported data, when applicable.
imports for a given site or not. Currently only Google Analytics is supported.
* `full_intervals` - map of dates indicating whether the interval has been * `full_intervals` - map of dates indicating whether the interval has been
cut off by the requested date range or not. For example, if looking at a cut off by the requested date range or not. For example, if looking at a
@ -86,7 +87,7 @@ defmodule PlausibleWeb.Api.StatsController do
"2021-11-01" => true, "2021-11-01" => true,
"2021-12-01" => false "2021-12-01" => false
}, },
"imports_exist" => false, "imported_source" => nil,
"interval" => "month", "interval" => "month",
"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"], "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
"plot" => [0, 0, 0, 0], "plot" => [0, 0, 0, 0],
@ -128,8 +129,9 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels) present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels) full_intervals = build_full_intervals(query, labels)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{ json(conn, %{
metric: metric,
plot: plot_timeseries(timeseries_result, metric), plot: plot_timeseries(timeseries_result, metric),
labels: labels, labels: labels,
comparison_plot: comparison_result && plot_timeseries(comparison_result, metric), comparison_plot: comparison_result && plot_timeseries(comparison_result, metric),
@ -137,7 +139,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index, present_index: present_index,
interval: query.interval, interval: query.interval,
with_imported: with_imported?(query, comparison_query), with_imported: with_imported?(query, comparison_query),
imports_exist: site.complete_import_ids != [], imported_source: site_import && SiteImport.label(site_import),
full_intervals: full_intervals full_intervals: full_intervals
}) })
else else
@ -212,12 +214,14 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query) {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{ json(conn, %{
top_stats: top_stats, top_stats: top_stats,
interval: query.interval, interval: query.interval,
sample_percent: sample_percent, sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query), with_imported: with_imported?(query, comparison_query),
imports_exist: site.complete_import_ids != [], imported_source: site_import && SiteImport.label(site_import),
comparing_from: comparison_query && comparison_query.date_range.first, comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last, comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first, from: query.date_range.first,
@ -295,12 +299,10 @@ defmodule PlausibleWeb.Api.StatsController do
}, },
%{ %{
name: "Unique conversions (last 30 min)", name: "Unique conversions (last 30 min)",
graph_metric: :visitors,
value: unique_conversions value: unique_conversions
}, },
%{ %{
name: "Total conversions (last 30 min)", name: "Total conversions (last 30 min)",
graph_metric: :events,
value: total_conversions value: total_conversions
} }
] ]
@ -323,12 +325,10 @@ defmodule PlausibleWeb.Api.StatsController do
}, },
%{ %{
name: "Unique visitors (last 30 min)", name: "Unique visitors (last 30 min)",
graph_metric: :visitors,
value: visitors value: visitors
}, },
%{ %{
name: "Pageviews (last 30 min)", name: "Pageviews (last 30 min)",
graph_metric: :pageviews,
value: pageviews value: pageviews
} }
] ]
@ -345,21 +345,15 @@ defmodule PlausibleWeb.Api.StatsController do
[ [
top_stats_entry(results, comparison, "Unique visitors", :total_visitors), top_stats_entry(results, comparison, "Unique visitors", :total_visitors),
top_stats_entry(results, comparison, "Unique conversions", :visitors, graphable?: true), top_stats_entry(results, comparison, "Unique conversions", :visitors),
top_stats_entry(results, comparison, "Total conversions", :events, graphable?: true), top_stats_entry(results, comparison, "Total conversions", :events),
on_full_build do on_full_build do
top_stats_entry(results, comparison, "Average revenue", :average_revenue, top_stats_entry(results, comparison, "Average revenue", :average_revenue, &format_money/1)
formatter: &format_money/1,
graphable?: true
)
end, end,
on_full_build do on_full_build do
top_stats_entry(results, comparison, "Total revenue", :total_revenue, top_stats_entry(results, comparison, "Total revenue", :total_revenue, &format_money/1)
formatter: &format_money/1,
graphable?: true
)
end, end,
top_stats_entry(results, comparison, "Conversion rate", :conversion_rate, graphable?: true) top_stats_entry(results, comparison, "Conversion rate", :conversion_rate)
] ]
|> Enum.reject(&is_nil/1) |> Enum.reject(&is_nil/1)
|> then(&{&1, 100}) |> then(&{&1, 100})
@ -393,63 +387,39 @@ defmodule PlausibleWeb.Api.StatsController do
stats = stats =
[ [
top_stats_entry(current_results, prev_results, "Unique visitors", :visitors, top_stats_entry(current_results, prev_results, "Unique visitors", :visitors),
graphable?: true top_stats_entry(current_results, prev_results, "Total visits", :visits),
), top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews),
top_stats_entry(current_results, prev_results, "Total visits", :visits, graphable?: true), top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit),
top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews, top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate),
graphable?: true top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration),
), top_stats_entry(current_results, prev_results, "Time on page", :time_on_page, fn
top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit, nil -> 0
graphable?: true value -> value
), end)
top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration,
graphable?: true
),
top_stats_entry(current_results, prev_results, "Time on page", :time_on_page,
formatter: fn
nil -> 0
value -> value
end
)
] ]
|> Enum.filter(& &1) |> Enum.filter(& &1)
{stats, current_results[:sample_percent][:value]} {stats, current_results[:sample_percent][:value]}
end end
defp top_stats_entry(current_results, prev_results, name, key, opts \\ []) do defp top_stats_entry(current_results, prev_results, name, key, formatter \\ & &1) do
if current_results[key] do if current_results[key] do
formatter = Keyword.get(opts, :formatter, & &1)
value = get_in(current_results, [key, :value]) value = get_in(current_results, [key, :value])
%{name: name, value: formatter.(value)} if prev_results do
|> maybe_put_graph_metric(opts, key) prev_value = get_in(prev_results, [key, :value])
|> maybe_put_comparison(prev_results, key, value, formatter) change = Stats.Compare.calculate_change(key, prev_value, value)
end
end
defp maybe_put_graph_metric(entry, opts, key) do %{
if Keyword.get(opts, :graphable?) do name: name,
entry |> Map.put(:graph_metric, key) value: formatter.(value),
else comparison_value: formatter.(prev_value),
entry change: change
end }
end else
%{name: name, value: formatter.(value)}
defp maybe_put_comparison(entry, prev_results, key, value, formatter) do end
if prev_results do
prev_value = get_in(prev_results, [key, :value])
change = Stats.Compare.calculate_change(key, prev_value, value)
entry
|> Map.put(:comparison_value, formatter.(prev_value))
|> Map.put(:change, change)
else
entry
end end
end end
@ -458,13 +428,6 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.from(site, params) query = Query.from(site, params)
pagination = parse_pagination(params) pagination = parse_pagination(params)
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:entry_page_hostname", query.filters["event:hostname"])
else
query
end
extra_metrics = extra_metrics =
if params["detailed"], do: [:bounce_rate, :visit_duration], else: [] if params["detailed"], do: [:bounce_rate, :visit_duration], else: []
@ -770,13 +733,6 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params) pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits, :visit_duration]) metrics = breakdown_metrics(query, [:visits, :visit_duration])
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:entry_page_hostname", query.filters["event:hostname"])
else
query
end
entry_pages = entry_pages =
Stats.breakdown(site, query, "visit:entry_page", metrics, pagination) Stats.breakdown(site, query, "visit:entry_page", metrics, pagination)
|> transform_keys(%{entry_page: :name}) |> transform_keys(%{entry_page: :name})
@ -807,13 +763,6 @@ defmodule PlausibleWeb.Api.StatsController do
{limit, page} = parse_pagination(params) {limit, page} = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits]) metrics = breakdown_metrics(query, [:visits])
query =
if query.filters["event:hostname"] do
Query.put_filter(query, "visit:exit_page_hostname", query.filters["event:hostname"])
else
query
end
exit_pages = exit_pages =
Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page}) Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page})
|> add_exit_rate(site, query, limit) |> add_exit_rate(site, query, limit)

@ -1,11 +1,6 @@
defmodule PlausibleWeb.GoogleAnalyticsController do defmodule PlausibleWeb.GoogleAnalyticsController do
use PlausibleWeb, :controller use PlausibleWeb, :controller
alias Plausible.Google
alias Plausible.Imported
require Plausible.Imported.SiteImport
plug(PlausibleWeb.RequireAccountPlug) plug(PlausibleWeb.RequireAccountPlug)
plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
@ -15,8 +10,6 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
"access_token" => access_token, "access_token" => access_token,
"refresh_token" => refresh_token, "refresh_token" => refresh_token,
"expires_at" => expires_at, "expires_at" => expires_at,
"start_date" => start_date,
"end_date" => end_date,
"legacy" => legacy "legacy" => legacy
}) do }) do
site = conn.assigns.site site = conn.assigns.site
@ -29,22 +22,17 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
access_token: access_token, access_token: access_token,
refresh_token: refresh_token, refresh_token: refresh_token,
expires_at: expires_at, expires_at: expires_at,
start_date: start_date,
end_date: end_date,
legacy: legacy, legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"} layout: {PlausibleWeb.LayoutView, "focus.html"}
) )
end end
def property_or_view_form( def property_or_view_form(conn, %{
conn, "access_token" => access_token,
%{ "refresh_token" => refresh_token,
"access_token" => access_token, "expires_at" => expires_at,
"refresh_token" => refresh_token, "legacy" => legacy
"expires_at" => expires_at, }) do
"legacy" => legacy
} = params
) do
site = conn.assigns.site site = conn.assigns.site
redirect_route = redirect_route =
@ -56,21 +44,9 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
result = result =
if legacy == "true" do if legacy == "true" do
Google.UA.API.list_views(access_token) Plausible.Google.UA.API.list_views(access_token)
else else
Google.API.list_properties_and_views(access_token) Plausible.Google.API.list_properties_and_views(access_token)
end
error =
case params["error"] do
"no_data" ->
"No data found. Nothing to import."
"no_time_window" ->
"Imported data time range is completely overlapping with existing data. Nothing to import."
_ ->
nil
end end
case result do case result do
@ -83,7 +59,6 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
expires_at: expires_at, expires_at: expires_at,
site: conn.assigns.site, site: conn.assigns.site,
properties_and_views: properties_and_views, properties_and_views: properties_and_views,
selected_property_or_view_error: error,
legacy: legacy, legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"} layout: {PlausibleWeb.LayoutView, "focus.html"}
) )
@ -96,175 +71,103 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
) )
|> redirect(external: redirect_route) |> redirect(external: redirect_route)
{:error, :timeout} ->
conn
|> put_flash(
:error,
"Google Analytics API has timed out. Please try again."
)
|> redirect(external: redirect_route)
{:error, _any} -> {:error, _any} ->
conn conn
|> put_flash( |> put_flash(
:error, :error,
"We were unable to list your Google Analytics properties and views. If the problem persists, please contact support for assistance." "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
) )
|> redirect(external: redirect_route) |> redirect(external: redirect_route)
end end
end end
# see https://stackoverflow.com/a/57416769 # see https://stackoverflow.com/a/57416769
@universal_analytics_new_user_metric_date ~D[2016-08-24] @google_analytics_new_user_metric_date ~D[2016-08-24]
def property_or_view(
conn,
%{
"property_or_view" => property_or_view,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
} = params
) do
site = conn.assigns.site
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
else
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
with {:ok, api_start_date} <-
Google.API.get_analytics_start_date(access_token, property_or_view),
{:ok, api_end_date} <- Google.API.get_analytics_end_date(access_token, property_or_view),
{:ok, start_date, end_date} <- Imported.check_dates(site, api_start_date, api_end_date) do
action =
if Timex.before?(api_start_date, @universal_analytics_new_user_metric_date) do
:user_metric_notice
else
:confirm
end
redirect(conn,
external:
Routes.google_analytics_path(conn, action, site.domain,
property_or_view: property_or_view,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
start_date: Date.to_iso8601(start_date),
end_date: Date.to_iso8601(end_date),
legacy: legacy
)
)
else
{:error, error} when error in [:no_data, :no_time_window] ->
params =
params
|> Map.take(["access_token", "refresh_token", "expires_at", "legacy"])
|> Map.put("error", Atom.to_string(error))
property_or_view_form(conn, params)
{:error, :authentication_failed} ->
conn
|> put_flash(
:error,
"Google Analytics authentication seems to have expired. Please try again."
)
|> redirect(external: redirect_route)
{:error, :timeout} ->
conn
|> put_flash(
:error,
"Google Analytics API has timed out. Please try again."
)
|> redirect(external: redirect_route)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
def confirm(conn, %{ def property_or_view(conn, %{
"property_or_view" => property_or_view, "property_or_view" => property_or_view,
"access_token" => access_token, "access_token" => access_token,
"refresh_token" => refresh_token, "refresh_token" => refresh_token,
"expires_at" => expires_at, "expires_at" => expires_at,
"start_date" => start_date,
"end_date" => end_date,
"legacy" => legacy "legacy" => legacy
}) do }) do
site = conn.assigns.site site = conn.assigns.site
start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view)
start_date = Date.from_iso8601!(start_date) case start_date do
end_date = Date.from_iso8601!(end_date) {:ok, nil} ->
{:ok, properties_and_views} =
redirect_route = if legacy == "true" do
if legacy == "true" do Plausible.Google.UA.API.list_views(access_token)
Routes.site_path(conn, :settings_integrations, site.domain) else
else Plausible.Google.API.list_properties_and_views(access_token)
Routes.site_path(conn, :settings_imports_exports, site.domain) end
end
case Google.API.get_property_or_view(access_token, property_or_view) do
{:ok, %{name: property_or_view_name, id: property_or_view}} ->
conn conn
|> assign(:skip_plausible_tracking, true) |> assign(:skip_plausible_tracking, true)
|> render("confirm.html", |> render("property_or_view_form.html",
access_token: access_token, access_token: access_token,
refresh_token: refresh_token, refresh_token: refresh_token,
expires_at: expires_at, expires_at: expires_at,
site: site, site: site,
selected_property_or_view: property_or_view, properties_and_views: properties_and_views,
selected_property_or_view_name: property_or_view_name, selected_property_or_view_error: "No data found. Nothing to import",
start_date: start_date,
end_date: end_date,
property?: Google.API.property?(property_or_view),
legacy: legacy, legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"} layout: {PlausibleWeb.LayoutView, "focus.html"}
) )
{:error, :authentication_failed} -> {:ok, date} ->
conn action =
|> put_flash( if Timex.before?(date, @google_analytics_new_user_metric_date) do
:error, :user_metric_notice
"Google Analytics authentication seems to have expired. Please try again." else
:confirm
end
redirect(conn,
to:
Routes.google_analytics_path(conn, action, site.domain,
property_or_view: property_or_view,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
) )
|> redirect(external: redirect_route) end
end
{:error, :timeout} -> def confirm(conn, %{
conn "property_or_view" => property_or_view,
|> put_flash( "access_token" => access_token,
:error, "refresh_token" => refresh_token,
"Google Analytics API has timed out. Please try again." "expires_at" => expires_at,
) "legacy" => legacy
|> redirect(external: redirect_route) }) do
site = conn.assigns.site
{:error, :not_found} -> start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view)
conn
|> put_flash(
:error,
"Google Analytics property not found. Please try again."
)
|> redirect(external: redirect_route)
{:error, _any} -> end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
conn
|> put_flash( {:ok, %{name: property_or_view_name, id: property_or_view}} =
:error, Plausible.Google.API.get_property_or_view(access_token, property_or_view)
"We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance."
) conn
|> redirect(external: redirect_route) |> assign(:skip_plausible_tracking, true)
end |> render("confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_property_or_view: property_or_view,
selected_property_or_view_name: property_or_view_name,
start_date: start_date,
end_date: end_date,
property?: Plausible.Google.API.property?(property_or_view),
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end end
def import(conn, %{ def import(conn, %{
@ -279,9 +182,6 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
site = conn.assigns.site site = conn.assigns.site
current_user = conn.assigns.current_user current_user = conn.assigns.current_user
start_date = Date.from_iso8601!(start_date)
end_date = Date.from_iso8601!(end_date)
redirect_route = redirect_route =
if legacy == "true" do if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain) Routes.site_path(conn, :settings_integrations, site.domain)
@ -289,48 +189,36 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
Routes.site_path(conn, :settings_imports_exports, site.domain) Routes.site_path(conn, :settings_imports_exports, site.domain)
end end
import_opts = [ if Plausible.Google.API.property?(property_or_view) do
label: property_or_view, {:ok, _} =
start_date: start_date, Plausible.Imported.GoogleAnalytics4.new_import(
end_date: end_date, site,
access_token: access_token, current_user,
refresh_token: refresh_token, property: property_or_view,
token_expires_at: expires_at, label: property_or_view,
legacy: legacy == "true" start_date: start_date,
] end_date: end_date,
access_token: access_token,
with {:ok, start_date, end_date} <- Imported.check_dates(site, start_date, end_date), refresh_token: refresh_token,
import_opts = [{:start_date, start_date}, {:end_date, end_date} | import_opts], token_expires_at: expires_at
{:ok, _} <- schedule_job(site, current_user, property_or_view, import_opts) do
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
else
{:error, :no_time_window} ->
conn
|> put_flash(
:error,
"Import failed. No data could be imported because date range overlaps with existing data."
)
|> redirect(external: redirect_route)
{:error, :import_in_progress} ->
conn
|> put_flash(
:error,
"There's another import still in progress. Please wait until it's completed or cancel it before starting a new one."
) )
|> redirect(external: redirect_route)
end
end
defp schedule_job(site, current_user, property_or_view, opts) do
if Google.API.property?(property_or_view) do
opts = Keyword.put(opts, :property, property_or_view)
Imported.GoogleAnalytics4.new_import(site, current_user, opts)
else else
opts = Keyword.put(opts, :view_id, property_or_view) Plausible.Imported.UniversalAnalytics.new_import(
Imported.UniversalAnalytics.new_import(site, current_user, opts) site,
current_user,
view_id: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
end end
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
end end
end end

@ -34,9 +34,9 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Mailer.send() |> Plausible.Mailer.send()
end end
redirect(conn, conn
external: Routes.site_path(conn, :add_snippet, site.domain, site_created: true) |> put_session(site.domain <> "_offer_email_report", true)
) |> redirect(external: Routes.site_path(conn, :add_snippet, site.domain))
{:error, {:over_limit, limit}} -> {:error, {:over_limit, limit}} ->
render(conn, "new.html", render(conn, "new.html",
@ -257,7 +257,7 @@ defmodule PlausibleWeb.SiteController do
end end
def settings_shields(conn, %{"shield" => shield}) def settings_shields(conn, %{"shield" => shield})
when shield in ["ip_addresses", "countries", "pages"] do when shield in ["ip_addresses", "countries"] do
site = conn.assigns.site site = conn.assigns.site
conn conn
@ -652,7 +652,7 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Site.remove_imported_data() |> Plausible.Site.remove_imported_data()
|> Repo.update!() |> Repo.update!()
site_import = Plausible.Imported.get_import(site, import_id) -> site_import = Plausible.Imported.get_import(import_id) ->
Oban.cancel_all_jobs( Oban.cancel_all_jobs(
from(j in Oban.Job, from(j in Oban.Job,
where: where:
@ -666,8 +666,6 @@ defmodule PlausibleWeb.SiteController do
Plausible.Repo.delete!(site_import) Plausible.Repo.delete!(site_import)
if site_import.legacy do if site_import.legacy do
Plausible.Purge.delete_imported_stats!(site, 0)
site site
|> Plausible.Site.remove_imported_data() |> Plausible.Site.remove_imported_data()
|> Repo.update!() |> Repo.update!()
@ -710,7 +708,7 @@ defmodule PlausibleWeb.SiteController do
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain)) |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end end
def csv_export(conn, _params) do def export(conn, _params) do
%{site: site, current_user: user} = conn.assigns %{site: site, current_user: user} = conn.assigns
Oban.insert!( Oban.insert!(
@ -727,15 +725,6 @@ defmodule PlausibleWeb.SiteController do
|> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain)) |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))
end end
def csv_import(conn, _params) do
conn
|> assign(:skip_plausible_tracking, true)
|> render("csv_import.html",
layout: {PlausibleWeb.LayoutView, "focus.html"},
connect_live_socket: true
)
end
def change_domain(conn, _params) do def change_domain(conn, _params) do
changeset = Plausible.Site.update_changeset(conn.assigns.site) changeset = Plausible.Site.update_changeset(conn.assigns.site)

@ -59,19 +59,22 @@ defmodule PlausibleWeb.StatsController do
cond do cond do
stats_start_date && can_see_stats? -> stats_start_date && can_see_stats? ->
offer_email_report = get_session(conn, site.domain <> "_offer_email_report")
conn conn
|> remove_email_report_banner(site)
|> put_resp_header("x-robots-tag", "noindex, nofollow") |> put_resp_header("x-robots-tag", "noindex, nofollow")
|> render("stats.html", |> render("stats.html",
site: site, site: site,
has_goals: Plausible.Sites.has_goals?(site), has_goals: Plausible.Sites.has_goals?(site),
revenue_goals: list_revenue_goals(site),
funnels: list_funnels(site), funnels: list_funnels(site),
has_props: Plausible.Props.configured?(site), has_props: Plausible.Props.configured?(site),
stats_start_date: stats_start_date, stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at), native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
title: title(conn, site), title: title(conn, site),
offer_email_report: offer_email_report,
demo: demo, demo: demo,
flags: get_flags(conn.assigns[:current_user], site), flags: get_flags(conn.assigns[:current_user]),
is_dbip: is_dbip(), is_dbip: is_dbip(),
dogfood_page_path: dogfood_page_path, dogfood_page_path: dogfood_page_path,
load_dashboard_js: true load_dashboard_js: true
@ -93,13 +96,10 @@ defmodule PlausibleWeb.StatsController do
defp list_funnels(site) do defp list_funnels(site) do
Plausible.Funnels.list(site) Plausible.Funnels.list(site)
end end
defp list_revenue_goals(site) do
Plausible.Goals.list_revenue_goals(site)
end
else else
defp list_funnels(_site), do: [] defp list_funnels(_site) do
defp list_revenue_goals(_site), do: [] []
end
end end
@doc """ @doc """
@ -323,19 +323,19 @@ defmodule PlausibleWeb.StatsController do
|> render("stats.html", |> render("stats.html",
site: shared_link.site, site: shared_link.site,
has_goals: Sites.has_goals?(shared_link.site), has_goals: Sites.has_goals?(shared_link.site),
revenue_goals: list_revenue_goals(shared_link.site),
funnels: list_funnels(shared_link.site), funnels: list_funnels(shared_link.site),
has_props: Plausible.Props.configured?(shared_link.site), has_props: Plausible.Props.configured?(shared_link.site),
stats_start_date: stats_start_date, stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at), native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: title(conn, shared_link.site), title: title(conn, shared_link.site),
offer_email_report: false,
demo: false, demo: false,
dogfood_page_path: "/share/:dashboard", dogfood_page_path: "/share/:dashboard",
shared_link_auth: shared_link.slug, shared_link_auth: shared_link.slug,
embedded: conn.params["embed"] == "true", embedded: conn.params["embed"] == "true",
background: conn.params["background"], background: conn.params["background"],
theme: conn.params["theme"], theme: conn.params["theme"],
flags: get_flags(conn.assigns[:current_user], shared_link.site), flags: get_flags(conn.assigns[:current_user]),
is_dbip: is_dbip(), is_dbip: is_dbip(),
load_dashboard_js: true load_dashboard_js: true
) )
@ -351,14 +351,18 @@ defmodule PlausibleWeb.StatsController do
end end
end end
defp remove_email_report_banner(conn, site) do
if conn.assigns[:current_user] do
delete_session(conn, site.domain <> "_offer_email_report")
else
conn
end
end
defp shared_link_cookie_name(slug), do: "shared-link-" <> slug defp shared_link_cookie_name(slug), do: "shared-link-" <> slug
defp get_flags(user, site) do defp get_flags(_user) do
%{ %{}
hostname_filter:
FunWithFlags.enabled?(:hostname_filter, for: user) ||
FunWithFlags.enabled?(:hostname_filter, for: site)
}
end end
defp is_dbip() do defp is_dbip() do

@ -1,228 +0,0 @@
defmodule PlausibleWeb.Live.CSVImport do
@moduledoc """
LiveView allowing uploading CSVs for imported tables to S3
"""
use PlausibleWeb, :live_view
alias Plausible.Imported.CSVImporter
@impl true
def mount(_params, session, socket) do
%{"site_id" => site_id, "user_id" => user_id} = session
socket =
socket
|> assign(site_id: site_id, user_id: user_id)
|> allow_upload(:import,
accept: [".csv", "text/csv"],
auto_upload: true,
max_entries: length(Plausible.Imported.tables()),
# 1GB
max_file_size: 1_000_000_000,
external: &presign_upload/2,
progress: &handle_progress/3
)
|> process_imported_tables()
{:ok, socket}
end
@impl true
def render(assigns) do
~H"""
<div>
<form action="#" method="post" phx-change="validate-upload-form" phx-submit="submit-upload-form">
<.csv_picker upload={@uploads.import} imported_tables={@imported_tables} />
<.confirm_button date_range={@date_range} can_confirm?={@can_confirm?} />
<p :for={error <- upload_errors(@uploads.import)} class="text-red-400">
<%= error_to_string(error) %>
</p>
</form>
</div>
"""
end
defp csv_picker(assigns) do
~H"""
<label
phx-drop-target={@upload.ref}
class="block border-2 dark:border-gray-600 rounded p-4 group hover:border-indigo-500 dark:hover:border-indigo-600 transition cursor-pointer"
>
<div class="flex items-center">
<div class="bg-gray-200 dark:bg-gray-600 rounded p-1 group-hover:bg-indigo-500 dark:group-hover:bg-indigo-600 transition">
<Heroicons.document_plus class="w-5 h-5 group-hover:text-white transition" />
</div>
<span class="ml-2 text-sm text-gray-600 dark:text-gray-500">
(or drag-and-drop your unzipped CSVs here)
</span>
<.live_file_input upload={@upload} class="hidden" />
</div>
<ul id="imported-tables" class="mt-3.5 mb-0.5 space-y-1.5">
<.imported_table
:for={{table, upload} <- @imported_tables}
table={table}
upload={upload}
errors={if(upload, do: upload_errors(@upload, upload), else: [])}
/>
</ul>
</label>
"""
end
defp confirm_button(assigns) do
~H"""
<button
type="submit"
disabled={not @can_confirm?}
class={[
"rounded-md w-full bg-indigo-600 px-3.5 py-2.5 text-sm font-semibold text-white shadow-sm hover:bg-indigo-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-400 dark:disabled:text-gray-400 dark:disabled:bg-gray-700 mt-4",
unless(@can_confirm?, do: "cursor-not-allowed")
]}
>
<%= if @date_range do %>
Confirm import from <%= @date_range.first %> to <%= @date_range.last %>
<% else %>
Confirm import
<% end %>
</button>
"""
end
defp imported_table(assigns) do
status =
cond do
assigns.upload && assigns.upload.progress == 100 -> :success
assigns.upload && assigns.upload.progress > 0 -> :in_progress
not Enum.empty?(assigns.errors) -> :error
true -> :empty
end
assigns = assign(assigns, status: status)
~H"""
<li id={@table} class="ml-1.5">
<div class="flex items-center space-x-2">
<Heroicons.document_check :if={@status == :success} class="w-4 h-4 text-indigo-600" />
<PlausibleWeb.Components.Generic.spinner
:if={@status == :in_progress}
class="w-4 h-4 text-indigo-600"
/>
<Heroicons.document :if={@status == :empty} class="w-4 h-4 text-gray-400 dark:text-gray-500" />
<Heroicons.document :if={@status == :error} class="w-4 h-4 text-red-600 dark:text-red-700" />
<span class={[
"text-sm",
if(@upload, do: "dark:text-gray-400", else: "text-gray-400 dark:text-gray-500"),
if(@status == :error, do: "text-red-600 dark:text-red-700")
]}>
<%= if @upload do %>
<%= @upload.client_name %>
<% else %>
<%= @table %>_YYYYMMDD_YYYYMMDD.csv
<% end %>
</span>
</div>
<p :for={error <- @errors} class="ml-6 text-sm text-red-600 dark:text-red-700">
<%= error_to_string(error) %>
</p>
</li>
"""
end
@impl true
def handle_event("validate-upload-form", _params, socket) do
{:noreply, process_imported_tables(socket)}
end
def handle_event("submit-upload-form", _params, socket) do
%{site_id: site_id, user_id: user_id, date_range: date_range} = socket.assigns
site = Plausible.Repo.get!(Plausible.Site, site_id)
user = Plausible.Repo.get!(Plausible.Auth.User, user_id)
uploads =
consume_uploaded_entries(socket, :import, fn meta, entry ->
{:ok, %{"s3_url" => meta.s3_url, "filename" => entry.client_name}}
end)
{:ok, _job} =
CSVImporter.new_import(site, user,
start_date: date_range.first,
end_date: date_range.last,
uploads: uploads
)
redirect_to =
Routes.site_path(socket, :settings_imports_exports, URI.encode_www_form(site.domain))
{:noreply, redirect(socket, external: redirect_to)}
end
defp error_to_string(:too_large), do: "is too large (max size is 1 gigabyte)"
defp error_to_string(:too_many_files), do: "too many files"
defp error_to_string(:not_accepted), do: "unacceptable file types"
defp error_to_string(:external_client_failure), do: "browser upload failed"
defp presign_upload(entry, socket) do
%{s3_url: s3_url, presigned_url: upload_url} =
Plausible.S3.import_presign_upload(socket.assigns.site_id, entry.client_name)
{:ok, %{uploader: "S3", s3_url: s3_url, url: upload_url}, socket}
end
defp handle_progress(:import, entry, socket) do
if entry.done? do
{:noreply, process_imported_tables(socket)}
else
{:noreply, socket}
end
end
defp process_imported_tables(socket) do
tables = Plausible.Imported.tables()
{completed, in_progress} = uploaded_entries(socket, :import)
{valid_uploads, invalid_uploads} =
Enum.split_with(completed ++ in_progress, &CSVImporter.valid_filename?(&1.client_name))
imported_tables_all_uploads =
Enum.map(tables, fn table ->
uploads =
Enum.filter(valid_uploads, fn upload ->
CSVImporter.extract_table(upload.client_name) == table
end)
{upload, replaced_uploads} = List.pop_at(uploads, -1)
{table, upload, replaced_uploads}
end)
imported_tables =
Enum.map(imported_tables_all_uploads, fn {table, upload, _replaced_uploads} ->
{table, upload}
end)
replaced_uploads =
Enum.flat_map(imported_tables_all_uploads, fn {_table, _upload, replaced_uploads} ->
replaced_uploads
end)
date_range = CSVImporter.date_range(Enum.map(valid_uploads, & &1.client_name))
all_uploaded? = completed != [] and in_progress == []
socket
|> cancel_uploads(invalid_uploads)
|> cancel_uploads(replaced_uploads)
|> assign(
imported_tables: imported_tables,
can_confirm?: all_uploaded?,
date_range: date_range
)
end
defp cancel_uploads(socket, uploads) do
Enum.reduce(uploads, socket, fn upload, socket ->
cancel_upload(socket, :import, upload.ref)
end)
end
end

@ -42,39 +42,7 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
end end
def render(assigns) do def render(assigns) do
import_in_progress? =
Enum.any?(
assigns.site_imports,
&(&1.live_status in [SiteImport.pending(), SiteImport.importing()])
)
assigns = assign(assigns, :import_in_progress?, import_in_progress?)
~H""" ~H"""
<div class="mt-5 flex gap-x-4">
<.button_link
class="w-36 h-20"
theme="bright"
disabled={@import_in_progress?}
href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
>
<img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
</.button_link>
<.button_link
class="w-36 h-20"
theme="bright"
disabled={@import_in_progress?}
href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</.button_link>
</div>
<p :if={@import_in_progress?} class="mt-4 text-red-400 text-sm">
No new imports can be started until the import in progress is completed or cancelled.
</p>
<header class="relative border-b border-gray-200 pb-4"> <header class="relative border-b border-gray-200 pb-4">
<h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100"> <h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">
Existing Imports Existing Imports
@ -94,11 +62,9 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
<li :for={entry <- @site_imports} class="py-4 flex items-center justify-between space-x-4"> <li :for={entry <- @site_imports} class="py-4 flex items-center justify-between space-x-4">
<div class="flex flex-col"> <div class="flex flex-col">
<p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100"> <p class="text-sm leading-5 font-medium text-gray-900 dark:text-gray-100">
<%= Plausible.Imported.SiteImport.label(entry.site_import) %> Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %>
<span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal"> <span :if={entry.live_status == SiteImport.completed()} class="text-xs font-normal">
(<%= PlausibleWeb.StatsView.large_number_format( (<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views)
Map.get(@pageview_counts, entry.site_import.id, 0)
) %> page views)
</span> </span>
<Heroicons.clock <Heroicons.clock
:if={entry.live_status == SiteImport.pending()} :if={entry.live_status == SiteImport.pending()}

@ -1,304 +0,0 @@
defmodule PlausibleWeb.Live.Shields.PageRules do
@moduledoc """
LiveView allowing page Rules management
"""
use Phoenix.LiveComponent, global_prefixes: ~w(x-)
use Phoenix.HTML
alias PlausibleWeb.Live.Components.Modal
alias Plausible.Shields
alias Plausible.Shield
import PlausibleWeb.ErrorHelpers
def update(assigns, socket) do
socket =
socket
|> assign(
page_rules_count: assigns[:page_rules_count] || socket.assigns.page_rules_count,
site: assigns[:site] || socket.assigns.site,
current_user: assigns[:current_user] || socket.assigns.current_user,
form: new_form()
)
|> assign_new(:page_rules, fn %{site: site} ->
Shields.list_page_rules(site)
end)
|> assign_new(:redundant_rules, fn %{page_rules: page_rules} ->
detect_redundancy(page_rules)
end)
{:ok, socket}
end
def render(assigns) do
~H"""
<section class="shadow bg-white dark:bg-gray-800 sm:rounded-md sm:overflow-hidden">
<div class="py-6 px-4 sm:p-6">
<header class="relative">
<h2 class="text-lg leading-6 font-medium text-gray-900 dark:text-gray-100">
Pages Block List
</h2>
<p class="mt-1 mb-4 text-sm leading-5 text-gray-500 dark:text-gray-200">
Reject incoming traffic for specific pages
</p>
<PlausibleWeb.Components.Generic.docs_info slug="top-pages#block-traffic-from-specific-pages-or-sections" />
</header>
<div class="border-t border-gray-200 pt-4 grid">
<div
:if={@page_rules_count < Shields.maximum_page_rules()}
class="mt-4 sm:ml-4 sm:mt-0 justify-self-end"
>
<PlausibleWeb.Components.Generic.button
id="add-page-rule"
x-data
x-on:click={Modal.JS.open("page-rule-form-modal")}
>
+ Add Page
</PlausibleWeb.Components.Generic.button>
</div>
<PlausibleWeb.Components.Generic.notice
:if={@page_rules_count >= Shields.maximum_page_rules()}
class="mt-4"
title="Maximum number of pages reached"
>
<p>
You've reached the maximum number of pages you can block (<%= Shields.maximum_page_rules() %>). Please remove one before adding another.
</p>
</PlausibleWeb.Components.Generic.notice>
</div>
<.live_component module={Modal} id="page-rule-form-modal">
<.form
:let={f}
for={@form}
phx-submit="save-page-rule"
phx-target={@myself}
class="max-w-md w-full mx-auto bg-white dark:bg-gray-800"
>
<h2 class="text-xl font-black dark:text-gray-100 mb-8">Add Page to Block List</h2>
<.live_component
submit_name="page_rule[page_path]"
submit_value={f[:page_path].value}
display_value={f[:page_path].value || ""}
module={PlausibleWeb.Live.Components.ComboBox}
suggest_fun={fn input, options -> suggest_page_paths(input, options, @site) end}
id={f[:page_path].id}
creatable
/>
<%= error_tag(f, :page_path) %>
<p class="text-sm mt-2 text-gray-500 dark:text-gray-200">
You can use a wildcard (<code>*</code>) to match multiple pages. For example,
<code>/blog/*</code>
will match <code>/blog/post</code>.
Once added, we will start rejecting traffic from this page within a few minutes.
</p>
<div class="py-4 mt-8">
<PlausibleWeb.Components.Generic.button type="submit" class="w-full">
Add Page →
</PlausibleWeb.Components.Generic.button>
</div>
</.form>
</.live_component>
<p
:if={Enum.empty?(@page_rules)}
class="text-sm text-gray-800 dark:text-gray-200 mt-12 mb-8 text-center"
>
No Page Rules configured for this Site.
</p>
<div
:if={not Enum.empty?(@page_rules)}
class="mt-8 overflow-hidden border-b border-gray-200 shadow dark:border-gray-900 sm:rounded-lg"
>
<table class="min-w-full divide-y divide-gray-200 dark:divide-gray-900">
<thead class="bg-gray-50 dark:bg-gray-900">
<tr>
<th
scope="col"
class="px-6 py-3 text-xs font-medium text-left text-gray-500 uppercase dark:text-gray-100"
>
page
</th>
<th
scope="col"
class="px-6 py-3 text-xs font-medium text-left text-gray-500 uppercase dark:text-gray-100"
>
Status
</th>
<th scope="col" class="px-6 py-3">
<span class="sr-only">Remove</span>
</th>
</tr>
</thead>
<tbody>
<%= for rule <- @page_rules do %>
<tr class="text-gray-900 dark:text-gray-100">
<td class="px-6 py-4 text-sm font-medium max-w-xs truncate text-ellipsis overflow-hidden">
<div class="flex items-center">
<span
id={"page-#{rule.id}"}
class="mr-4 cursor-help border-b border-dotted border-gray-400 text-ellipsis overflow-hidden"
title={"#{rule.page_path}\n\nAdded at #{format_added_at(rule.inserted_at, @site.timezone)} by #{rule.added_by}"}
>
<%= rule.page_path %>
</span>
</div>
</td>
<td class="px-6 py-4 text-sm text-gray-500">
<div class="flex items-center">
<span :if={rule.action == :deny}>
Blocked
</span>
<span :if={rule.action == :allow}>
Allowed
</span>
<span
:if={@redundant_rules[rule.id]}
title={"This rule might be redundant because the following rules may match first:\n\n#{Enum.join(@redundant_rules[rule.id], "\n")}"}
class="pl-4"
>
<Heroicons.exclamation_triangle class="h-4 w-4 text-red-500" />
</span>
</div>
</td>
<td class="px-6 py-4 text-sm font-medium text-right">
<button
id={"remove-page-rule-#{rule.id}"}
phx-target={@myself}
phx-click="remove-page-rule"
phx-value-rule-id={rule.id}
class="text-sm text-red-600"
data-confirm="Are you sure you want to revoke this rule?"
>
Remove
</button>
</td>
</tr>
<% end %>
</tbody>
</table>
</div>
</div>
</section>
"""
end
def handle_event("save-page-rule", %{"page_rule" => params}, socket) do
user = socket.assigns.current_user
case Shields.add_page_rule(
socket.assigns.site.id,
params,
added_by: user
) do
{:ok, rule} ->
page_rules = [rule | socket.assigns.page_rules]
socket =
socket
|> Modal.close("page-rule-form-modal")
|> assign(
form: new_form(),
page_rules: page_rules,
page_rules_count: socket.assigns.page_rules_count + 1,
redundant_rules: detect_redundancy(page_rules)
)
# Make sure to clear the combobox input after adding a page rule, on subsequent modal reopening
send_update(PlausibleWeb.Live.Components.ComboBox,
id: "page_rule_page_code",
display_value: ""
)
send_flash(
:success,
"Page rule added successfully. Traffic will be rejected within a few minutes."
)
{:noreply, socket}
{:error, %Ecto.Changeset{} = changeset} ->
{:noreply, assign(socket, form: to_form(changeset))}
end
end
def handle_event("remove-page-rule", %{"rule-id" => rule_id}, socket) do
Shields.remove_page_rule(socket.assigns.site.id, rule_id)
send_flash(
:success,
"Page rule removed successfully. Traffic will be resumed within a few minutes."
)
page_rules = Enum.reject(socket.assigns.page_rules, &(&1.id == rule_id))
{:noreply,
socket
|> assign(
page_rules_count: socket.assigns.page_rules_count - 1,
page_rules: page_rules,
redundant_rules: detect_redundancy(page_rules)
)}
end
def send_flash(kind, message) do
send(self(), {:flash, kind, message})
end
defp new_form() do
%Shield.PageRule{}
|> Shield.PageRule.changeset(%{})
|> to_form()
end
defp format_added_at(dt, tz) do
dt
|> Plausible.Timezones.to_datetime_in_timezone(tz)
|> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{m}:{s}")
end
def suggest_page_paths(input, _options, site) do
query = Plausible.Stats.Query.from(site, %{})
site
|> Plausible.Stats.filter_suggestions(query, "page", input)
|> Enum.map(fn %{label: label, value: value} -> {label, value} end)
end
defp detect_redundancy(page_rules) do
page_rules
|> Enum.reduce(%{}, fn rule, acc ->
{[^rule], remaining_rules} =
Enum.split_with(
page_rules,
fn r -> r == rule end
)
conflicting =
remaining_rules
|> Enum.filter(fn candidate ->
rule
|> Map.fetch!(:page_path_pattern)
|> maybe_compile()
|> Regex.match?(candidate.page_path)
end)
|> Enum.map(& &1.id)
Enum.reduce(conflicting, acc, fn conflicting_rule_id, acc ->
Map.update(acc, conflicting_rule_id, [rule.page_path], fn existing ->
[rule.page_path | existing]
end)
end)
end)
end
defp maybe_compile(pattern) when is_binary(pattern), do: Regex.compile!(pattern)
defp maybe_compile(%Regex{} = pattern), do: pattern
end

@ -1,53 +0,0 @@
defmodule PlausibleWeb.Live.Shields.Pages do
@moduledoc """
LiveView for IP Addresses Shield
"""
use PlausibleWeb, :live_view
use Phoenix.HTML
alias Plausible.Shields
alias Plausible.Sites
def mount(
_params,
%{
"domain" => domain,
"current_user_id" => user_id
},
socket
) do
socket =
socket
|> assign_new(:site, fn ->
Sites.get_for_user!(user_id, domain, [:owner, :admin, :super_admin])
end)
|> assign_new(:page_rules_count, fn %{site: site} ->
Shields.count_page_rules(site)
end)
|> assign_new(:current_user, fn ->
Plausible.Repo.get(Plausible.Auth.User, user_id)
end)
{:ok, socket}
end
def render(assigns) do
~H"""
<div>
<.flash_messages flash={@flash} />
<.live_component
module={PlausibleWeb.Live.Shields.PageRules}
current_user={@current_user}
page_rules_count={@page_rules_count}
site={@site}
id="page-rules-#{@current_user.id}"
/>
</div>
"""
end
def handle_info({:flash, kind, message}, socket) do
socket = put_live_flash(socket, kind, message)
{:noreply, socket}
end
end

@ -390,8 +390,7 @@ defmodule PlausibleWeb.Router do
delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
post "/:website/settings/export", SiteController, :csv_export post "/:website/settings/export", SiteController, :export
get "/:website/settings/import", SiteController, :csv_import
get "/:domain/export", StatsController, :csv_export get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats get "/:domain/*path", StatsController, :stats

@ -6,8 +6,22 @@
<%= link("Click here", to: @link) %> to view your dashboard. <%= link("Click here", to: @link) %> to view your dashboard.
<% else %> <% else %>
Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that! Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
<br /><br /> <br /> <br />
Please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need. This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
<br /> <br />
Please log into your Google Analytics account and go to Admin -> Property Settings -> User Analysis. Then make sure that "Enable User Metric in Reporting" is set to OFF. Then try importing again. You can switch the setting back after the import.
<br /> <br /> If you've already changed that setting, then it is something else: <br />
<ul>
<li>
Do you have any stats in the Google Analytics property you tried to import? We only import the stats from your first Google Analytics visitor until your first Plausible Analytics visitor. The import fails if there's no returned data from Google from before you counted your first Plausible visitor.
</li>
<li>
Did you set strict data retention limits in your Google Analytics account? You can check the data retention limit you have set for your property in your Google Analytics admin settings by visiting the "Tracking Info" section and clicking on "Data Retention".
</li>
</ul>
<br />
If all of the above is fine, please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<%= if full_build?() do %> <%= if full_build?() do %>
<br /> <br /> <br /> <br />
Please reply to this email to let us know if you're still experiencing issues with the import. Please reply to this email to let us know if you're still experiencing issues with the import.

@ -6,47 +6,55 @@
<%= hidden_input(f, :expires_at, value: @expires_at) %> <%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= hidden_input(f, :legacy, value: @legacy) %> <%= hidden_input(f, :legacy, value: @legacy) %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200"> <%= case @start_date do %>
Stats from this <% {:ok, start_date} -> %>
<%= if @property? do %> <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
property Stats from this
<% else %> <%= if @property? do %>
view property
<% end %> <% else %>
and time period will be imported from your Google Analytics account to your Plausible dashboard view
</div> <% end %>
and time period will be imported from your Google Analytics account to your Plausible dashboard
</div>
<div class="mt-6"> <div class="mt-6">
<%= styled_label( <%= styled_label(
f, f,
:property_or_view, :property_or_view,
"Google Analytics #{if @property?, do: "property", else: "view"}" "Google Analytics #{if @property?, do: "property", else: "view"}"
) %> ) %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= @selected_property_or_view_name %> <%= @selected_property_or_view_name %>
</span> </span>
<%= hidden_input(f, :property_or_view, <%= hidden_input(f, :property_or_view,
readonly: "true", readonly: "true",
value: @selected_property_or_view value: @selected_property_or_view
) %> ) %>
</div> </div>
<div class="flex justify-between mt-3"> <div class="flex justify-between mt-3">
<div class="w-36"> <div class="w-36">
<%= styled_label(f, :start_date, "From") %> <%= styled_label(f, :start_date, "From") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@start_date) %> <%= PlausibleWeb.EmailView.date_format(start_date) %>
</span> </span>
<%= hidden_input(f, :start_date, value: @start_date, readonly: "true") %> <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
</div> </div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div> <div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36"> <div class="w-36">
<%= styled_label(f, :end_date, "To") %> <%= styled_label(f, :end_date, "To") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@end_date) %> <%= PlausibleWeb.EmailView.date_format(@end_date) %>
</span> </span>
<%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %> <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div> </div>
</div> </div>
<% {:error, error} -> %>
<p class="text-gray-700 dark:text-gray-300 mt-6">
The following error occurred when fetching your Google Analytics data.
</p>
<p class="text-red-700 font-medium mt-3"><%= error %></p>
<% end %>
<%= submit("Confirm import", class: "button mt-6") %> <%= submit("Confirm import", class: "button mt-6") %>
<% end %> <% end %>

@ -39,8 +39,6 @@
access_token: @access_token, access_token: @access_token,
refresh_token: @refresh_token, refresh_token: @refresh_token,
expires_at: @expires_at, expires_at: @expires_at,
start_date: @start_date,
end_date: @end_date,
legacy: @legacy legacy: @legacy
), ),
class: "button mt-6" class: "button mt-6"

@ -1,20 +0,0 @@
<div class="max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded p-6 mb-4 mt-8">
<h2 class="text-xl font-black dark:text-gray-100">Import from CSV files</h2>
<div class="my-3 space-y-1.5 text-sm text-gray-400">
<p>
Please ensure each file follows
<.styled_link href="https://plausible.io/docs/csv-import">
our CSV format guidelines.
</.styled_link>
</p>
<p>
You can upload multiple files simultaneously by either selecting them in the file dialog or dragging and dropping them into the designated area.
</p>
</div>
<%= live_render(@conn, PlausibleWeb.Live.CSVImport,
session: %{"site_id" => @site.id, "user_id" => @current_user.id}
) %>
</div>

@ -10,6 +10,24 @@
<PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" /> <PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" />
</header> </header>
<div class="mt-5 flex gap-x-4">
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20"
theme="bright"
href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
>
<img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
</PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20 opacity-40 cursor-not-allowed"
theme="bright"
href=""
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</PlausibleWeb.Components.Generic.button_link>
</div>
<%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings, <%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,
session: %{"domain" => @site.domain} session: %{"domain" => @site.domain}
) %> ) %>

@ -14,11 +14,4 @@
"domain" => @site.domain "domain" => @site.domain
} }
) %> ) %>
<% "pages" -> %>
<%= live_render(@conn, PlausibleWeb.Live.Shields.Pages,
session: %{
"site_id" => @site.id,
"domain" => @site.domain
}
) %>
<% end %> <% end %>

@ -1,8 +1,3 @@
<PlausibleWeb.Components.FirstDashboardLaunchBanner.set
:if={@conn.params["site_created"] == "true"}
site={@site}
/>
<div class="w-full max-w-3xl mt-4 mx-auto flex"> <div class="w-full max-w-3xl mt-4 mx-auto flex">
<%= form_for @conn, "/", [class: "max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> <%= form_for @conn, "/", [class: "max-w-lg w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-bold dark:text-gray-100">Add JavaScript snippet</h2> <h2 class="text-xl font-bold dark:text-gray-100">Add JavaScript snippet</h2>

@ -28,11 +28,11 @@
Please subscribe to the appropriate tier with the link below to access the stats again. Please subscribe to the appropriate tier with the link below to access the stats again.
</p> </p>
<p class="mt-6 text-sm text-gray-500"> <p class="mt-6 text-sm text-gray-500">
You can configure your You can check the
<.styled_link href={Routes.site_path(@conn, :settings_general, @site.domain)}> <.styled_link href={Routes.site_path(@conn, :settings_general, @site.domain)}>
site settings site settings
</.styled_link> </.styled_link>
but your access to the dashboard is restricted. and we're still counting the stats but your access to the dashboard is restricted.
</p> </p>
</div> </div>
<div class="mt-6 w-full text-center"> <div class="mt-6 w-full text-center">

@ -0,0 +1,63 @@
<div class="<%= stats_container_class(@conn) %>" data-site-domain="<%= @site.domain %>">
<%= if @offer_email_report do %>
<div class="w-full px-4 text-sm font-bold text-center text-blue-900 bg-blue-200 rounded transition" style="top: 91px" role="alert">
<%= link("Team members, email reports and GA import. Explore more →", to: "/#{URI.encode_www_form(@site.domain)}/settings/email-reports", class: "py-2 block") %>
</div>
<% end %>
<%= if @site.locked do %>
<div class="w-full px-4 py-4 text-sm font-bold text-center text-yellow-800 bg-yellow-100 rounded transition" style="top: 91px" role="alert">
<p>This dashboard is actually locked. You are viewing it with super-admin access</p>
</div>
<% end %>
<div class="pt-6"></div>
<div
id="stats-react-container"
style="overflow-anchor: none;"
data-domain="<%= @site.domain %>"
data-offset="<%= Plausible.Site.tz_offset(@site) %>"
data-has-goals="<%= @has_goals %>"
data-conversions-opted-out="<%= Plausible.Billing.Feature.Goals.opted_out?(@site) %>"
data-funnels-opted-out="<%= Plausible.Billing.Feature.Funnels.opted_out?(@site) %>"
data-props-opted-out="<%= Plausible.Billing.Feature.Props.opted_out?(@site) %>"
data-funnels-available="<%= Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok %>"
data-props-available="<%= Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok %>"
data-funnels="<%= Jason.encode!(@funnels) %>"
data-has-props="<%= @has_props %>"
data-logged-in="<%= !!@conn.assigns[:current_user] %>"
data-stats-begin="<%= @stats_start_date %>"
data-native-stats-begin="<%= @native_stats_start_date %>"
data-shared-link-auth="<%= assigns[:shared_link_auth] %>"
data-embedded="<%= @conn.assigns[:embedded] %>"
data-background="<%= @conn.assigns[:background] %>"
data-is-dbip="<%= @is_dbip %>"
data-current-user-role="<%= @conn.assigns[:current_user_role] %>"
data-flags="<%= Jason.encode!(@flags) %>"
data-valid-intervals-by-period="<%= Plausible.Stats.Interval.valid_by_period(site: @site) |> Jason.encode!() %>">
</div>
<div id="modal_root"></div>
<%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
<div class="bg-gray-50 dark:bg-gray-850">
<div class="py-12 lg:py-16 lg:flex lg:items-center lg:justify-between">
<h2 class="text-3xl font-extrabold tracking-tight text-gray-900 leading-9 sm:text-4xl sm:leading-10 dark:text-gray-100">
Want these stats for your website?
<br />
<span class="text-indigo-600">Start your free trial today.</span>
</h2>
<div class="flex mt-8 lg:flex-shrink-0 lg:mt-0">
<div class="inline-flex shadow rounded-md">
<a href="/register" class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-white bg-indigo-600 border border-transparent leading-6 rounded-md hover:bg-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out">
Get started
</a>
</div>
<div class="inline-flex ml-3 shadow rounded-md">
<a href="/" class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-indigo-600 bg-white border border-transparent leading-6 rounded-md dark:text-gray-100 dark:bg-gray-800 hover:text-indigo-500 dark:hover:text-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out">
Learn more
</a>
</div>
</div>
</div>
</div>
<% end %>
</div>

@ -1,76 +0,0 @@
<div class={stats_container_class(@conn)} data-site-domain={@site.domain}>
<PlausibleWeb.Components.FirstDashboardLaunchBanner.render site={@site} />
<%= if @site.locked do %>
<div
class="w-full px-4 py-4 text-sm font-bold text-center text-yellow-800 bg-yellow-100 rounded transition"
style="top: 91px"
role="alert"
>
<p>This dashboard is actually locked. You are viewing it with super-admin access</p>
</div>
<% end %>
<div class="pt-6"></div>
<div
id="stats-react-container"
style="overflow-anchor: none;"
data-domain={@site.domain}
data-offset={Plausible.Site.tz_offset(@site)}
data-has-goals={to_string(@has_goals)}
data-conversions-opted-out={to_string(Plausible.Billing.Feature.Goals.opted_out?(@site))}
data-funnels-opted-out={to_string(Plausible.Billing.Feature.Funnels.opted_out?(@site))}
data-props-opted-out={to_string(Plausible.Billing.Feature.Props.opted_out?(@site))}
data-funnels-available={
to_string(Plausible.Billing.Feature.Funnels.check_availability(@site.owner) == :ok)
}
data-props-available={
to_string(Plausible.Billing.Feature.Props.check_availability(@site.owner) == :ok)
}
data-revenue-goals={Jason.encode!(@revenue_goals)}
data-funnels={Jason.encode!(@funnels)}
data-has-props={to_string(@has_props)}
data-logged-in={to_string(!!@conn.assigns[:current_user])}
data-stats-begin={@stats_start_date}
data-native-stats-begin={@native_stats_start_date}
data-shared-link-auth={assigns[:shared_link_auth]}
data-embedded={to_string(@conn.assigns[:embedded])}
data-background={@conn.assigns[:background]}
data-is-dbip={to_string(@is_dbip)}
data-current-user-role={@conn.assigns[:current_user_role]}
data-flags={Jason.encode!(@flags)}
data-valid-intervals-by-period={
Plausible.Stats.Interval.valid_by_period(site: @site) |> Jason.encode!()
}
>
</div>
<div id="modal_root"></div>
<%= if !@conn.assigns[:current_user] && @conn.assigns[:demo] do %>
<div class="bg-gray-50 dark:bg-gray-850">
<div class="py-12 lg:py-16 lg:flex lg:items-center lg:justify-between">
<h2 class="text-3xl font-extrabold tracking-tight text-gray-900 leading-9 sm:text-4xl sm:leading-10 dark:text-gray-100">
Want these stats for your website? <br />
<span class="text-indigo-600">Start your free trial today.</span>
</h2>
<div class="flex mt-8 lg:flex-shrink-0 lg:mt-0">
<div class="inline-flex shadow rounded-md">
<a
href="/register"
class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-white bg-indigo-600 border border-transparent leading-6 rounded-md hover:bg-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out"
>
Get started
</a>
</div>
<div class="inline-flex ml-3 shadow rounded-md">
<a
href="/"
class="inline-flex items-center justify-center px-5 py-3 text-base font-medium text-indigo-600 bg-white border border-transparent leading-6 rounded-md dark:text-gray-100 dark:bg-gray-800 hover:text-indigo-500 dark:hover:text-indigo-500 focus:outline-none focus:ring transition duration-150 ease-in-out"
>
Learn more
</a>
</div>
</div>
</div>
</div>
<% end %>
</div>

@ -65,15 +65,10 @@ defmodule PlausibleWeb.LayoutView do
%{ %{
key: "Shields", key: "Shields",
icon: :shield_exclamation, icon: :shield_exclamation,
value: value: [
[ %{key: "IP Addresses", value: "shields/ip_addresses"},
%{key: "IP Addresses", value: "shields/ip_addresses"}, %{key: "Countries", value: "shields/countries"}
%{key: "Countries", value: "shields/countries"}, ]
if FunWithFlags.enabled?(:shield_pages, for: conn.assigns[:site]) do
%{key: "Pages", value: "shields/pages"}
end
]
|> Enum.reject(&is_nil/1)
}, },
%{key: "Email Reports", value: "email-reports", icon: :envelope}, %{key: "Email Reports", value: "email-reports", icon: :envelope},
if conn.assigns[:current_user_role] == :owner do if conn.assigns[:current_user_role] == :owner do

@ -147,7 +147,7 @@
"timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"},
"tls_certificate_check": {:hex, :tls_certificate_check, "1.21.0", "042ab2c0c860652bc5cf69c94e3a31f96676d14682e22ec7813bd173ceff1788", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6cee6cffc35a390840d48d463541d50746a7b0e421acaadb833cfc7961e490e7"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.21.0", "042ab2c0c860652bc5cf69c94e3a31f96676d14682e22ec7813bd173ceff1788", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6cee6cffc35a390840d48d463541d50746a7b0e421acaadb833cfc7961e490e7"},
"tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"},
"ua_inspector": {:hex, :ua_inspector, "3.9.0", "2021bbddb1ee41f202da7f006fb09f5c5617ad579108b7b9bcf1881828462f04", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0f2d1b9806e78b14b91b6f20eaaa4a68f8989f4f2a99a376b874c295ac50a30d"}, "ua_inspector": {:hex, :ua_inspector, "3.8.0", "c0b0d13200a9bd509225f15ea8cf275c0bec27390a21c355746ff8b8a88c3e4d", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "7c980bae82a4754075b933e0f383935a681e5a2628856ad3ecf6eb80d8139539"},
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
"uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"},
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},

@ -1,35 +0,0 @@
defmodule Plausible.IngestRepo.Migrations.AddMetricsToImportedTables do
use Ecto.Migration
@add_pageviews_for [
"imported_browsers",
"imported_devices",
"imported_entry_pages",
"imported_exit_pages",
"imported_locations",
"imported_operating_systems",
"imported_sources"
]
def up do
for table <- @add_pageviews_for do
add_column(table, "pageviews", "UInt64")
end
add_column("imported_browsers", "browser_version", "String")
add_column("imported_exit_pages", "bounces", "UInt32")
add_column("imported_exit_pages", "visit_duration", "UInt64")
add_column("imported_operating_systems", "operating_system_version", "String")
add_column("imported_pages", "visits", "UInt64")
add_column("imported_sources", "referrer", "String")
add_column("imported_sources", "utm_source", "String")
end
defp add_column(table, column, type) do
execute """
ALTER TABLE #{table}
#{Plausible.MigrationUtils.on_cluster_statement(table)}
ADD COLUMN #{column} #{type}
"""
end
end

@ -1,19 +0,0 @@
defmodule Plausible.ClickhouseRepo.Migrations.HostnamesInSessions do
use Ecto.Migration
def up do
execute """
ALTER TABLE sessions_v2
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
ADD COLUMN exit_page_hostname String CODEC(ZSTD(3))
"""
end
def down do
execute """
ALTER TABLE sessions_v2
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
DROP COLUMN exit_page_hostname
"""
end
end

@ -11,8 +11,6 @@
# and so on) as they will fail if something goes wrong. # and so on) as they will fail if something goes wrong.
FunWithFlags.enable(:imports_exports) FunWithFlags.enable(:imports_exports)
FunWithFlags.enable(:shield_pages)
FunWithFlags.enable(:hostname_filter)
user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible")
@ -162,7 +160,7 @@ native_stats_range
[ [
site_id: site.id, site_id: site.id,
hostname: Enum.random(["en.dummy.site", "es.dummy.site", "dummy.site"]), hostname: site.domain,
timestamp: put_random_time.(date, index), timestamp: put_random_time.(date, index),
referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]), referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]), browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]),
@ -197,7 +195,7 @@ native_stats_range
[ [
name: goal4.event_name, name: goal4.event_name,
site_id: site.id, site_id: site.id,
hostname: Enum.random(["en.dummy.site", "es.dummy.site", "dummy.site"]), hostname: site.domain,
timestamp: put_random_time.(date, index), timestamp: put_random_time.(date, index),
referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]), referrer_source: Enum.random(["", "Facebook", "Twitter", "DuckDuckGo", "Google"]),
browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]), browser: Enum.random(["Edge", "Chrome", "Safari", "Firefox", "Vivaldi"]),

File diff suppressed because it is too large Load Diff

@ -8,13 +8,13 @@
- regex: 'NetFront' - regex: 'NetFront'
name: 'NetFront' name: 'NetFront'
- regex: 'Edge/' - regex: 'Edge'
name: 'Edge' name: 'Edge'
- regex: 'Trident' - regex: 'Trident'
name: 'Trident' name: 'Trident'
- regex: 'Chrome/(?!1?\d\.|2[0-7]\.)' - regex: '(?<!SmartHu)Blink'
name: 'Blink' name: 'Blink'
- regex: '(?:Apple)?WebKit' - regex: '(?:Apple)?WebKit'

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save