Compare commits
22 Commits
master
...
journey-re
| Author | SHA1 | Date |
|---|---|---|
|
|
9676d3ebd1 | 3 years ago |
|
|
7e8fa24f8f | 3 years ago |
|
|
641e3f636c | 3 years ago |
|
|
5b228d65bc | 3 years ago |
|
|
4e1ffba5f5 | 3 years ago |
|
|
b8f2c14332 | 3 years ago |
|
|
1f1a6d96b7 | 3 years ago |
|
|
0aada2632e | 3 years ago |
|
|
79d427a4a6 | 3 years ago |
|
|
39aef3447b | 3 years ago |
|
|
dd19966600 | 3 years ago |
|
|
04d01cc860 | 3 years ago |
|
|
b5d6c917a9 | 3 years ago |
|
|
d60a18dc12 | 3 years ago |
|
|
00b9213d0c | 3 years ago |
|
|
61cdfd2529 | 3 years ago |
|
|
2fcf5a0bb4 | 3 years ago |
|
|
70063bc79e | 3 years ago |
|
|
1bf2be88d3 | 3 years ago |
|
|
be6aa3ef4c | 3 years ago |
|
|
1033334279 | 3 years ago |
|
|
0f95999eb1 | 3 years ago |
@ -1,28 +0,0 @@
|
||||
name: Validate migrations
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'priv/repo/migrations/**'
|
||||
- 'priv/ingest_repo/migrations/**'
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: App code does not change at the same time
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: changes
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
lib:
|
||||
- 'lib/**'
|
||||
- 'extra/**'
|
||||
- 'config/**'
|
||||
|
||||
- if: steps.changes.outputs.lib == 'true'
|
||||
run: |
|
||||
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
|
||||
exit 1
|
||||
@ -1,247 +0,0 @@
|
||||
import React from 'react';
|
||||
import { withRouter } from 'react-router-dom'
|
||||
import Chart from 'chart.js/auto';
|
||||
import { navigateToQuery } from '../../query'
|
||||
import GraphTooltip from './graph-tooltip'
|
||||
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
|
||||
import dateFormatter from './date-formatter';
|
||||
import FadeIn from '../../fade-in';
|
||||
import classNames from 'classnames';
|
||||
|
||||
const calculateMaximumY = function(dataset) {
|
||||
const yAxisValues = dataset
|
||||
.flatMap((item) => item.data)
|
||||
.map((item) => item || 0)
|
||||
|
||||
if (yAxisValues) {
|
||||
return Math.max(...yAxisValues)
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
class LineGraph extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.regenerateChart = this.regenerateChart.bind(this);
|
||||
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
|
||||
}
|
||||
|
||||
getGraphMetric() {
|
||||
let metric = this.props.graphData.metric
|
||||
|
||||
if (metric == 'visitors' && this.props.query.filters.goal) {
|
||||
return 'conversions'
|
||||
} else {
|
||||
return metric
|
||||
}
|
||||
}
|
||||
|
||||
regenerateChart() {
|
||||
const { graphData, query } = this.props
|
||||
const metric = this.getGraphMetric()
|
||||
const graphEl = document.getElementById("main-graph-canvas")
|
||||
this.ctx = graphEl.getContext('2d');
|
||||
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
|
||||
|
||||
return new Chart(this.ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: graphData.labels,
|
||||
datasets: dataSet
|
||||
},
|
||||
options: {
|
||||
animation: false,
|
||||
plugins: {
|
||||
legend: { display: false },
|
||||
tooltip: {
|
||||
enabled: false,
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
position: 'average',
|
||||
external: GraphTooltip(graphData, metric, query)
|
||||
},
|
||||
},
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
onResize: this.updateWindowDimensions,
|
||||
elements: { line: { tension: 0 }, point: { radius: 0 } },
|
||||
onClick: this.maybeHopToHoveredPeriod.bind(this),
|
||||
scale: {
|
||||
ticks: { precision: 0, maxTicksLimit: 8 }
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
ticks: {
|
||||
callback: METRIC_FORMATTER[metric],
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
},
|
||||
grid: {
|
||||
zeroLineColor: 'transparent',
|
||||
drawBorder: false,
|
||||
}
|
||||
},
|
||||
yComparison: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
display: false,
|
||||
grid: { display: false },
|
||||
},
|
||||
x: {
|
||||
grid: { display: false },
|
||||
ticks: {
|
||||
callback: function(val, _index, _ticks) {
|
||||
if (this.getLabelForValue(val) == "__blank__") return ""
|
||||
|
||||
const hasMultipleYears =
|
||||
graphData.labels
|
||||
.filter((date) => typeof date === 'string')
|
||||
.map(date => date.split('-')[0])
|
||||
.filter((value, index, list) => list.indexOf(value) === index)
|
||||
.length > 1
|
||||
|
||||
if (graphData.interval === 'hour' && query.period !== 'day') {
|
||||
const date = dateFormatter({
|
||||
interval: "date",
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
const hour = dateFormatter({
|
||||
interval: graphData.interval,
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
// Returns a combination of date and hour. This is because
|
||||
// small intervals like hour may return multiple days
|
||||
// depending on the query period.
|
||||
return `${date}, ${hour}`
|
||||
}
|
||||
|
||||
if (graphData.interval === 'minute' && query.period !== 'realtime') {
|
||||
return dateFormatter({
|
||||
interval: "hour", longForm: false, period: query.period,
|
||||
})(this.getLabelForValue(val))
|
||||
}
|
||||
|
||||
return dateFormatter({
|
||||
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
},
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
}
|
||||
}
|
||||
},
|
||||
interaction: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
repositionTooltip(e) {
|
||||
const tooltipEl = document.getElementById('chartjs-tooltip');
|
||||
if (tooltipEl && window.innerWidth >= 768) {
|
||||
if (e.clientX > 0.66 * window.innerWidth) {
|
||||
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
|
||||
tooltipEl.style.left = null;
|
||||
} else {
|
||||
tooltipEl.style.right = null;
|
||||
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
|
||||
}
|
||||
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
|
||||
tooltipEl.style.opacity = 1;
|
||||
}
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
if (this.props.graphData) {
|
||||
this.chart = this.regenerateChart();
|
||||
}
|
||||
window.addEventListener('mousemove', this.repositionTooltip);
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps) {
|
||||
const { graphData, darkTheme } = this.props;
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
|
||||
if (
|
||||
graphData !== prevProps.graphData ||
|
||||
darkTheme !== prevProps.darkTheme
|
||||
) {
|
||||
|
||||
if (graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
this.chart = this.regenerateChart();
|
||||
this.chart.update();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (!graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
// Ensure that the tooltip doesn't hang around when we are loading more data
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
if (tooltip) {
|
||||
tooltip.style.opacity = 0;
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
window.removeEventListener('mousemove', this.repositionTooltip)
|
||||
}
|
||||
|
||||
/**
|
||||
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
|
||||
* @param {*} chart - The chart instance.
|
||||
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
|
||||
*/
|
||||
updateWindowDimensions(chart, dimensions) {
|
||||
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
|
||||
}
|
||||
|
||||
maybeHopToHoveredPeriod(e) {
|
||||
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
|
||||
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
|
||||
|
||||
if (this.props.graphData.interval === 'month') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
|
||||
} else if (this.props.graphData.interval === 'date') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
const { graphData } = this.props
|
||||
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
|
||||
|
||||
return (
|
||||
<FadeIn show={graphData}>
|
||||
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
|
||||
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
|
||||
</div>
|
||||
</FadeIn>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default withRouter(LineGraph)
|
||||
@ -1,17 +0,0 @@
|
||||
import React from "react"
|
||||
|
||||
export default function SamplingNotice({topStatData}) {
|
||||
const samplePercent = topStatData?.samplePercent
|
||||
|
||||
if (samplePercent && samplePercent < 100) {
|
||||
return (
|
||||
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
|
||||
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -1,53 +0,0 @@
|
||||
import React, { useState } from "react"
|
||||
import * as api from '../../api'
|
||||
import { getCurrentInterval } from "./interval-picker"
|
||||
|
||||
export default function StatsExport({site, query}) {
|
||||
const [exporting, setExporting] = useState(false)
|
||||
|
||||
function startExport() {
|
||||
setExporting(true)
|
||||
document.cookie = "exporting="
|
||||
pollExportReady()
|
||||
}
|
||||
|
||||
function pollExportReady() {
|
||||
if (document.cookie.includes('exporting')) {
|
||||
setTimeout(pollExportReady, 1000)
|
||||
} else {
|
||||
setExporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
function renderLoading() {
|
||||
return (
|
||||
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function renderExportLink() {
|
||||
const interval = getCurrentInterval(site, query)
|
||||
const queryParams = api.serializeQuery(query, [{ interval }])
|
||||
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
|
||||
|
||||
return (
|
||||
<a href={endpoint} download onClick={startExport}>
|
||||
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
|
||||
<polyline points="7 10 12 15 17 10"></polyline>
|
||||
<line x1="12" y1="15" x2="12" y2="3"></line>
|
||||
</svg>
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-4 h-4 mx-2">
|
||||
{exporting && renderLoading()}
|
||||
{!exporting && renderExportLink()}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
import React from "react"
|
||||
import { parseNaiveDate, isBefore } from '../../util/date'
|
||||
import { Link } from 'react-router-dom'
|
||||
import * as url from '../../util/url'
|
||||
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
|
||||
|
||||
export default function WithImportedSwitch({site, topStatData}) {
|
||||
if (!topStatData?.imports_exist) {
|
||||
return null
|
||||
}
|
||||
|
||||
function isBeforeNativeStats(date) {
|
||||
if (!date) return false
|
||||
|
||||
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
|
||||
const parsedDate = parseNaiveDate(date)
|
||||
|
||||
return isBefore(parsedDate, nativeStatsBegin, "day")
|
||||
}
|
||||
|
||||
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
|
||||
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
|
||||
|
||||
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
|
||||
const withImported = topStatData.with_imported;
|
||||
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
|
||||
const target = url.setQuery('with_imported', !withImported)
|
||||
const tip = withImported ? "" : "do not ";
|
||||
|
||||
return (
|
||||
<Link to={target} className="w-4 h-4 mx-2">
|
||||
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
|
||||
<BarsArrowUpIcon className={"absolute " + toggleColor} />
|
||||
</div>
|
||||
</Link>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240302"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
{
|
||||
"account": "accounts/28425555",
|
||||
"createTime": "2024-02-22T10:50:15.462Z",
|
||||
"currencyCode": "USD",
|
||||
"displayName": "account.one - GA4",
|
||||
"name": "properties/428685444",
|
||||
"parent": "accounts/28425555",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY",
|
||||
"serviceLevel": "GOOGLE_ANALYTICS_STANDARD",
|
||||
"timeZone": "Europe/Warsaw",
|
||||
"updateTime": "2024-02-22T10:50:15.462Z"
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
{
|
||||
"accountSummaries": [
|
||||
{
|
||||
"account": "accounts/28425178",
|
||||
"displayName": "account.one",
|
||||
"name": "accountSummaries/28425178",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "account.one - GA4",
|
||||
"parent": "accounts/28425178",
|
||||
"property": "properties/428685906",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"account": "accounts/45336102",
|
||||
"displayName": "account.two",
|
||||
"name": "accountSummaries/45336102"
|
||||
},
|
||||
{
|
||||
"account": "accounts/54516992",
|
||||
"displayName": "Demo Account",
|
||||
"name": "accountSummaries/54516992",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "GA4 - Flood-It!",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/153293282",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
},
|
||||
{
|
||||
"displayName": "GA4 - Google Merch Shop",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/213025502",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,786 +0,0 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Amsterdam"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "totalUsers",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "bounces",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "sessions",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "userEngagementDuration",
|
||||
"type": "TYPE_SECONDS"
|
||||
}
|
||||
],
|
||||
"rowCount": 31,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240131"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "141"
|
||||
},
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "56"
|
||||
},
|
||||
{
|
||||
"value": "146"
|
||||
},
|
||||
{
|
||||
"value": "4815"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240130"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "42"
|
||||
},
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "3481"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240129"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "105"
|
||||
},
|
||||
{
|
||||
"value": "153"
|
||||
},
|
||||
{
|
||||
"value": "54"
|
||||
},
|
||||
{
|
||||
"value": "113"
|
||||
},
|
||||
{
|
||||
"value": "2895"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240128"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "184"
|
||||
},
|
||||
{
|
||||
"value": "259"
|
||||
},
|
||||
{
|
||||
"value": "74"
|
||||
},
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "6861"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240127"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "202"
|
||||
},
|
||||
{
|
||||
"value": "67"
|
||||
},
|
||||
{
|
||||
"value": "158"
|
||||
},
|
||||
{
|
||||
"value": "5259"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240126"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "132"
|
||||
},
|
||||
{
|
||||
"value": "189"
|
||||
},
|
||||
{
|
||||
"value": "57"
|
||||
},
|
||||
{
|
||||
"value": "142"
|
||||
},
|
||||
{
|
||||
"value": "5545"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240125"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "151"
|
||||
},
|
||||
{
|
||||
"value": "229"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "5365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240124"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "137"
|
||||
},
|
||||
{
|
||||
"value": "221"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "148"
|
||||
},
|
||||
{
|
||||
"value": "4454"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240123"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "166"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "4850"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240122"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "161"
|
||||
},
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "73"
|
||||
},
|
||||
{
|
||||
"value": "165"
|
||||
},
|
||||
{
|
||||
"value": "4871"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240121"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "254"
|
||||
},
|
||||
{
|
||||
"value": "397"
|
||||
},
|
||||
{
|
||||
"value": "114"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "8845"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240120"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "214"
|
||||
},
|
||||
{
|
||||
"value": "383"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "6914"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240119"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "381"
|
||||
},
|
||||
{
|
||||
"value": "5571"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "228"
|
||||
},
|
||||
{
|
||||
"value": "98549"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240118"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "177"
|
||||
},
|
||||
{
|
||||
"value": "278"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "186"
|
||||
},
|
||||
{
|
||||
"value": "5553"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240117"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "204"
|
||||
},
|
||||
{
|
||||
"value": "271"
|
||||
},
|
||||
{
|
||||
"value": "97"
|
||||
},
|
||||
{
|
||||
"value": "216"
|
||||
},
|
||||
{
|
||||
"value": "6777"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240116"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "200"
|
||||
},
|
||||
{
|
||||
"value": "280"
|
||||
},
|
||||
{
|
||||
"value": "65"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9381"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240115"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "255"
|
||||
},
|
||||
{
|
||||
"value": "311"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "12365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240114"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "243"
|
||||
},
|
||||
{
|
||||
"value": "302"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "257"
|
||||
},
|
||||
{
|
||||
"value": "11073"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240113"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "231"
|
||||
},
|
||||
{
|
||||
"value": "276"
|
||||
},
|
||||
{
|
||||
"value": "92"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "10084"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240112"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "215"
|
||||
},
|
||||
{
|
||||
"value": "265"
|
||||
},
|
||||
{
|
||||
"value": "75"
|
||||
},
|
||||
{
|
||||
"value": "225"
|
||||
},
|
||||
{
|
||||
"value": "10195"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240111"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "183"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "8200"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240110"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9056"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240109"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "201"
|
||||
},
|
||||
{
|
||||
"value": "290"
|
||||
},
|
||||
{
|
||||
"value": "63"
|
||||
},
|
||||
{
|
||||
"value": "213"
|
||||
},
|
||||
{
|
||||
"value": "11651"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240108"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "209"
|
||||
},
|
||||
{
|
||||
"value": "7946"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240107"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "205"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "80"
|
||||
},
|
||||
{
|
||||
"value": "218"
|
||||
},
|
||||
{
|
||||
"value": "10048"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240106"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "256"
|
||||
},
|
||||
{
|
||||
"value": "303"
|
||||
},
|
||||
{
|
||||
"value": "119"
|
||||
},
|
||||
{
|
||||
"value": "281"
|
||||
},
|
||||
{
|
||||
"value": "8440"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240105"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "226"
|
||||
},
|
||||
{
|
||||
"value": "270"
|
||||
},
|
||||
{
|
||||
"value": "83"
|
||||
},
|
||||
{
|
||||
"value": "237"
|
||||
},
|
||||
{
|
||||
"value": "9311"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240104"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "81"
|
||||
},
|
||||
{
|
||||
"value": "236"
|
||||
},
|
||||
{
|
||||
"value": "12051"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240103"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "283"
|
||||
},
|
||||
{
|
||||
"value": "79"
|
||||
},
|
||||
{
|
||||
"value": "245"
|
||||
},
|
||||
{
|
||||
"value": "10628"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240102"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"value": "210"
|
||||
},
|
||||
{
|
||||
"value": "11168"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240101"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "224"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "197"
|
||||
},
|
||||
{
|
||||
"value": "8080"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240222"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20200421"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20120118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20170118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,136 +0,0 @@
|
||||
defmodule Plausible.Cache.Adapter do
|
||||
@moduledoc """
|
||||
Interface for the underlying cache implementation.
|
||||
Currently: ConCache
|
||||
|
||||
Using the Adapter module directly, the user must ensure that the relevant
|
||||
processes are available to use, which is normally done via the child specification.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()
|
||||
def child_spec(name, child_id, opts \\ [])
|
||||
when is_atom(name) and is_atom(child_id) and is_list(opts) do
|
||||
cache_name = Keyword.get(opts, :cache_name, name)
|
||||
child_id = Keyword.get(opts, :child_id, child_id)
|
||||
ttl_check_interval = Keyword.get(opts, :ttl_check_interval, false)
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Keyword.put(:name, cache_name)
|
||||
|> Keyword.put(:ttl_check_interval, ttl_check_interval)
|
||||
|
||||
Supervisor.child_spec(
|
||||
{ConCache, opts},
|
||||
id: child_id
|
||||
)
|
||||
end
|
||||
|
||||
@spec size(atom()) :: non_neg_integer() | nil
|
||||
def size(cache_name) do
|
||||
ConCache.size(cache_name)
|
||||
catch
|
||||
:exit, _ -> nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any()) :: any()
|
||||
def get(cache_name, key) do
|
||||
result = ConCache.get(cache_name, key)
|
||||
if is_nil(result), do: miss(cache_name), else: hit(cache_name)
|
||||
result
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any(), (-> any())) :: any()
|
||||
def get(cache_name, key, fallback_fn) do
|
||||
cache = ConCache.Owner.cache(cache_name)
|
||||
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
get_or_store_isolated(cache, cache_name, key, fallback_fn)
|
||||
|
||||
value ->
|
||||
hit(cache_name)
|
||||
value
|
||||
end
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put(atom(), any(), any()) :: any()
|
||||
def put(cache_name, key, value) do
|
||||
:ok = ConCache.put(cache_name, key, value)
|
||||
value
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting a key to '#{cache_name}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put_many(atom(), [any()]) :: :ok
|
||||
def put_many(cache_name, items) when is_list(items) do
|
||||
true = :ets.insert(ConCache.ets(cache_name), items)
|
||||
:ok
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting keys to '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec delete(atom(), any()) :: :ok
|
||||
def delete(cache_name, key) do
|
||||
ConCache.dirty_delete(cache_name, key)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error deleting a key in '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec keys(atom()) :: Enumerable.t()
|
||||
def keys(cache_name) do
|
||||
ets = ConCache.ets(cache_name)
|
||||
|
||||
Stream.resource(
|
||||
fn -> :ets.first(ets) end,
|
||||
fn
|
||||
:"$end_of_table" -> {:halt, nil}
|
||||
prev_key -> {[prev_key], :ets.next(ets, prev_key)}
|
||||
end,
|
||||
fn _ -> :ok end
|
||||
)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
[]
|
||||
end
|
||||
|
||||
defp hit(cache_name) do
|
||||
Plausible.Cache.Stats.record_hit(cache_name)
|
||||
end
|
||||
|
||||
defp miss(cache_name) do
|
||||
Plausible.Cache.Stats.record_miss(cache_name)
|
||||
end
|
||||
|
||||
defp get_or_store_isolated(cache, cache_name, key, fun) do
|
||||
ConCache.isolated(cache_name, key, fn ->
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
new_value = fun.()
|
||||
ConCache.Operations.dirty_put(cache, key, new_value)
|
||||
miss(cache_name)
|
||||
new_value
|
||||
|
||||
existing ->
|
||||
hit(cache_name)
|
||||
existing
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -1,84 +0,0 @@
|
||||
defmodule Plausible.Cache.Stats do
|
||||
@moduledoc """
|
||||
Keeps track of hit/miss ratio for various caches.
|
||||
"""
|
||||
|
||||
use GenServer
|
||||
|
||||
@hit :hit
|
||||
@miss :miss
|
||||
@telemetry_hit [:plausible, :cache, :adapter, @hit]
|
||||
@telemetry_miss [:plausible, :cache, :adapter, @miss]
|
||||
@telemetry_events [@telemetry_hit, @telemetry_miss]
|
||||
|
||||
def start_link(_opts) do
|
||||
GenServer.start_link(__MODULE__, nil)
|
||||
end
|
||||
|
||||
def record_hit(cache_name) do
|
||||
:telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def record_miss(cache_name) do
|
||||
:telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def init(nil) do
|
||||
__MODULE__ =
|
||||
:ets.new(__MODULE__, [
|
||||
:public,
|
||||
:named_table,
|
||||
:set,
|
||||
read_concurrency: true,
|
||||
write_concurrency: true
|
||||
])
|
||||
|
||||
:telemetry.attach_many(
|
||||
"plausible-cache-stats",
|
||||
@telemetry_events,
|
||||
&__MODULE__.handle_telemetry_event/4,
|
||||
nil
|
||||
)
|
||||
|
||||
{:ok, nil}
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @hit)
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @miss)
|
||||
end
|
||||
|
||||
def gather(cache_name) do
|
||||
{:ok,
|
||||
%{
|
||||
hit_rate: hit_rate(cache_name),
|
||||
count: size(cache_name) || 0
|
||||
}}
|
||||
end
|
||||
|
||||
defdelegate size(cache_name), to: Plausible.Cache.Adapter
|
||||
|
||||
def bump(cache_name, type) do
|
||||
:ets.update_counter(
|
||||
__MODULE__,
|
||||
{cache_name, type},
|
||||
1,
|
||||
{{cache_name, type}, 0}
|
||||
)
|
||||
end
|
||||
|
||||
def hit_rate(cache_name) do
|
||||
hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)
|
||||
miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)
|
||||
hit_miss = hit + miss
|
||||
|
||||
if hit_miss == 0 do
|
||||
0.0
|
||||
else
|
||||
hit / hit_miss * 100
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,188 +0,0 @@
|
||||
defmodule Plausible.DataMigration.SiteImports do
|
||||
@moduledoc """
|
||||
Site imports migration backfilling SiteImport entries for old imports
|
||||
and alters import end dates to match actual end date of respective import stats.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.ClickhouseRepo
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
def run(opts \\ []) do
|
||||
dry_run? = Keyword.get(opts, :dry_run?, true)
|
||||
|
||||
site_import_query =
|
||||
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
|
||||
|
||||
sites_with_imports =
|
||||
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|
||||
|> Repo.all(log: false)
|
||||
|
||||
sites_count = length(sites_with_imports)
|
||||
|
||||
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
|
||||
|
||||
for {site, idx} <- Enum.with_index(sites_with_imports) do
|
||||
site_imports =
|
||||
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all(log: false)
|
||||
|
||||
IO.puts(
|
||||
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
|
||||
)
|
||||
|
||||
site_imports =
|
||||
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
|
||||
IO.puts("Creating legacy site import entry for site ID #{site.id}")
|
||||
|
||||
# create legacy entry if there's not one yet
|
||||
params =
|
||||
site.imported_data
|
||||
|> Imported.SiteImport.from_legacy()
|
||||
|> Map.put(:site_id, site.id)
|
||||
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
|
||||
|
||||
legacy_site_import =
|
||||
%Imported.SiteImport{}
|
||||
|> Ecto.Changeset.change(params)
|
||||
|> insert!(dry_run?)
|
||||
|
||||
[legacy_site_import | site_imports]
|
||||
else
|
||||
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
|
||||
|
||||
site_imports
|
||||
end
|
||||
|
||||
# adjust end date for each site import
|
||||
for site_import <- site_imports do
|
||||
IO.puts(
|
||||
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
|
||||
)
|
||||
|
||||
import_ids =
|
||||
if site_import.legacy do
|
||||
[0, site_import.id]
|
||||
else
|
||||
[site_import.id]
|
||||
end
|
||||
|
||||
end_date = imported_stats_end_date(site.id, import_ids)
|
||||
|
||||
if !end_date do
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
|
||||
)
|
||||
|
||||
if site_import.legacy do
|
||||
clear_imported_data(site, dry_run?)
|
||||
end
|
||||
|
||||
delete!(site_import, dry_run?)
|
||||
else
|
||||
end_date =
|
||||
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
|
||||
end_date
|
||||
else
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
|
||||
)
|
||||
|
||||
site_import.end_date
|
||||
end
|
||||
|
||||
site_import
|
||||
|> Ecto.Changeset.change(end_date: end_date)
|
||||
|> update!(dry_run?)
|
||||
|
||||
IO.puts(
|
||||
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Done processing site ID #{site.id}")
|
||||
end
|
||||
|
||||
IO.puts("Finished")
|
||||
end
|
||||
|
||||
# Exposed for testing purposes
|
||||
@doc false
|
||||
def imported_stats_end_date(site_id, import_ids) do
|
||||
[first_schema | schemas] = Imported.schemas()
|
||||
|
||||
query =
|
||||
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
|
||||
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
|
||||
end)
|
||||
|
||||
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
|
||||
|
||||
if dates != [] do
|
||||
case Enum.max(dates, Date) do
|
||||
# no stats for this domain yet
|
||||
~D[1970-01-01] ->
|
||||
nil
|
||||
|
||||
date ->
|
||||
date
|
||||
end
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp insert!(changeset, false = _dry_run?) do
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
defp insert!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
changeset
|
||||
|> Ecto.Changeset.change(id: 0)
|
||||
|> Ecto.Changeset.apply_changes()
|
||||
else
|
||||
raise "Invalid insert: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, false = _dry_run?) do
|
||||
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, true = _dry_run?) do
|
||||
%{site | imported_data: nil}
|
||||
end
|
||||
|
||||
defp update!(changeset, false = _dry_run?) do
|
||||
Repo.update!(changeset)
|
||||
end
|
||||
|
||||
defp update!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
Ecto.Changeset.apply_changes(changeset)
|
||||
else
|
||||
raise "Invalid update: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp delete!(entity, false = _dry_run?) do
|
||||
Repo.delete!(entity)
|
||||
end
|
||||
|
||||
defp delete!(entity, true = _dry_run?) do
|
||||
entity
|
||||
end
|
||||
|
||||
defp max_date_query(schema, site_id, import_ids) do
|
||||
from(q in schema,
|
||||
where: q.site_id == ^site_id,
|
||||
where: q.import_id in ^import_ids,
|
||||
select: %{max_date: fragment("max(?)", q.date)}
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -1,371 +0,0 @@
|
||||
defmodule Plausible.Exports do
|
||||
@moduledoc """
|
||||
Contains functions to export data for events and sessions as Zip archives.
|
||||
"""
|
||||
|
||||
require Plausible
|
||||
import Ecto.Query
|
||||
|
||||
@doc """
|
||||
Renders filename for the Zip archive containing the exported CSV files.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"plausible_io_20210101_20241231.zip"
|
||||
|
||||
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"Bücher_example_20210101_20241231.zip"
|
||||
|
||||
"""
|
||||
def archive_filename(domain, min_date, max_date) do
|
||||
name =
|
||||
Enum.join(
|
||||
[
|
||||
String.replace(domain, ".", "_"),
|
||||
Calendar.strftime(min_date, "%Y%m%d"),
|
||||
Calendar.strftime(max_date, "%Y%m%d")
|
||||
],
|
||||
"_"
|
||||
)
|
||||
|
||||
name <> ".zip"
|
||||
end
|
||||
|
||||
@doc """
|
||||
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
|
||||
tables into the format of `imported_*` tables for a website.
|
||||
"""
|
||||
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
|
||||
%{String.t() => Ecto.Query.t()}
|
||||
def export_queries(site_id, opts \\ []) do
|
||||
extname = opts[:extname] || ".csv"
|
||||
date_range = opts[:date_range]
|
||||
|
||||
filename = fn table ->
|
||||
name =
|
||||
if date_range do
|
||||
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
|
||||
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
|
||||
"#{table}_#{first_date}_#{last_date}"
|
||||
else
|
||||
table
|
||||
end
|
||||
|
||||
name <> extname
|
||||
end
|
||||
|
||||
%{
|
||||
filename.("imported_visitors") => export_visitors_q(site_id),
|
||||
filename.("imported_sources") => export_sources_q(site_id),
|
||||
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
|
||||
filename.("imported_pages") => export_pages_q(site_id),
|
||||
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
|
||||
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
|
||||
filename.("imported_locations") => export_locations_q(site_id),
|
||||
filename.("imported_devices") => export_devices_q(site_id),
|
||||
filename.("imported_browsers") => export_browsers_q(site_id),
|
||||
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
|
||||
}
|
||||
end
|
||||
|
||||
Plausible.on_full_build do
|
||||
defp sampled(table) do
|
||||
Plausible.Stats.Sampling.add_query_hint(from(table))
|
||||
end
|
||||
else
|
||||
defp sampled(table) do
|
||||
table
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop date(timestamp) do
|
||||
quote do
|
||||
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visit_duration(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
|
||||
:visit_duration
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visitors(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
|
||||
:visitors
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visits(t) do
|
||||
quote do
|
||||
selected_as(sum(unquote(t).sign), :visits)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop bounces(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
|
||||
:bounces
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop pageviews(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
|
||||
:pageviews
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_visitors_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
visitors(s),
|
||||
pageviews(s),
|
||||
bounces(s),
|
||||
visits(s),
|
||||
visit_duration(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_sources_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [
|
||||
selected_as(:date),
|
||||
selected_as(:source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term
|
||||
],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.referrer_source, :source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term,
|
||||
pageviews(s),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_pages_q(site_id) do
|
||||
window_q =
|
||||
from e in sampled("events_v2"),
|
||||
where: e.site_id == ^site_id,
|
||||
select: %{
|
||||
timestamp: e.timestamp,
|
||||
next_timestamp:
|
||||
over(fragment("leadInFrame(?)", e.timestamp),
|
||||
partition_by: e.session_id,
|
||||
order_by: e.timestamp,
|
||||
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
|
||||
),
|
||||
pathname: e.pathname,
|
||||
hostname: e.hostname,
|
||||
name: e.name,
|
||||
user_id: e.user_id,
|
||||
session_id: e.session_id,
|
||||
_sample_factor: fragment("_sample_factor")
|
||||
}
|
||||
|
||||
from e in subquery(window_q),
|
||||
group_by: [selected_as(:date), e.pathname],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(e.timestamp),
|
||||
selected_as(fragment("any(?)", e.hostname), :hostname),
|
||||
selected_as(e.pathname, :page),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
|
||||
:visits
|
||||
),
|
||||
visitors(e),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
),
|
||||
# NOTE: are exits pageviews or any events?
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
|
||||
:exits
|
||||
),
|
||||
selected_as(
|
||||
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
|
||||
:time_on_page
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_entry_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.entry_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.entry_page,
|
||||
visitors(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:entrances
|
||||
),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_exit_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.exit_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.exit_page,
|
||||
visitors(s),
|
||||
visit_duration(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:exits
|
||||
),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_locations_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
|
||||
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.country_code, :country),
|
||||
selected_as(s.subdivision1_code, :region),
|
||||
selected_as(s.city_geoname_id, :city),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_devices_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.screen_size],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.screen_size, :device),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_browsers_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.browser, s.browser_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.browser,
|
||||
s.browser_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_operating_systems_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.operating_system,
|
||||
s.operating_system_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Creates a streamable Zip archive from the provided (named) Ecto queries.
|
||||
|
||||
Example usage:
|
||||
|
||||
{:ok, pool} = Ch.start_link(pool_size: 1)
|
||||
|
||||
DBConnection.run(pool, fn conn ->
|
||||
conn
|
||||
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!("export.zip"))
|
||||
|> Stream.run()
|
||||
end)
|
||||
|
||||
"""
|
||||
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
|
||||
Enumerable.t()
|
||||
def stream_archive(conn, named_queries, opts \\ []) do
|
||||
entries =
|
||||
Enum.map(named_queries, fn {name, query} ->
|
||||
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
|
||||
|
||||
datastream =
|
||||
conn
|
||||
|> Ch.stream(sql, params, opts)
|
||||
|> Stream.map(fn %Ch.Result{data: data} -> data end)
|
||||
|
||||
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
|
||||
end)
|
||||
|
||||
Zstream.zip(entries)
|
||||
end
|
||||
end
|
||||
@ -1,152 +0,0 @@
|
||||
defmodule Plausible.Google.GA4.API do
|
||||
@moduledoc """
|
||||
API for Google Analytics 4.
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.GA4
|
||||
|
||||
require Logger
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 50_000
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
def list_properties(access_token) do
|
||||
case GA4.HTTP.list_accounts_for_user(access_token) do
|
||||
{:ok, %{"accountSummaries" => accounts}} ->
|
||||
accounts =
|
||||
accounts
|
||||
|> Enum.filter(& &1["propertySummaries"])
|
||||
|> Enum.map(fn account ->
|
||||
%{"account" => account_id, "displayName" => account_name} = account
|
||||
|
||||
{"#{account_name} (#{account_id})",
|
||||
Enum.map(account["propertySummaries"], fn property ->
|
||||
%{"displayName" => property_name, "property" => property_id} = property
|
||||
|
||||
{"#{property_name} (#{property_id})", property_id}
|
||||
end)}
|
||||
end)
|
||||
|
||||
{:ok, accounts}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, lookup_property) do
|
||||
case GA4.HTTP.get_property(access_token, lookup_property) do
|
||||
{:ok, property} ->
|
||||
%{"displayName" => property_name, "name" => property_id, "account" => account_id} =
|
||||
property
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
id: property_id,
|
||||
name: "#{property_name} (#{property_id})",
|
||||
account_id: account_id
|
||||
}}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_start_date(access_token, property)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_end_date(access_token, property)
|
||||
end
|
||||
|
||||
def import_analytics(date_range, property, auth, persist_fn) do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
|
||||
)
|
||||
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, property, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, property, access_token, persist_fn) do
|
||||
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
|
||||
)
|
||||
|
||||
report_request = prepare_request(report_request, date_range, property, access_token)
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case GA4.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, row_count}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
if report_request.offset + @per_page < row_count do
|
||||
fetch_and_persist(
|
||||
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
|
||||
)
|
||||
|
||||
{:error, cause}
|
||||
else
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
|
||||
)
|
||||
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp prepare_request(report_request, date_range, property, access_token) do
|
||||
%GA4.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
property: property,
|
||||
access_token: access_token,
|
||||
offset: 0,
|
||||
limit: @per_page
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -1,244 +0,0 @@
|
||||
defmodule Plausible.Google.GA4.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Google Analytics 4 API.
|
||||
"""
|
||||
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
require Logger
|
||||
|
||||
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
|
||||
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: report_request.property,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
|
||||
metrics: Enum.map(report_request.metrics, &build_metric/1),
|
||||
orderBys: [
|
||||
%{
|
||||
dimension: %{
|
||||
dimensionName: "date",
|
||||
orderType: "ALPHANUMERIC"
|
||||
},
|
||||
desc: true
|
||||
}
|
||||
],
|
||||
limit: report_request.limit,
|
||||
offset: report_request.offset
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url =
|
||||
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
url,
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
row_count <- Map.fetch!(report, "rowCount"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, row_count}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
|
||||
)
|
||||
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp build_metric(expression) do
|
||||
case String.split(expression, " = ") do
|
||||
[name, expression] ->
|
||||
%{
|
||||
name: name,
|
||||
expression: expression
|
||||
}
|
||||
|
||||
[name] ->
|
||||
%{name: name}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"rows" => rows,
|
||||
"dimensionHeaders" => dimension_headers,
|
||||
"metricHeaders" => metric_headers
|
||||
})
|
||||
when is_list(rows) do
|
||||
dimension_headers = Enum.map(dimension_headers, & &1["name"])
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
|
||||
dimension_values = Enum.map(dimensions, & &1["value"])
|
||||
metric_values = Enum.map(metrics, & &1["value"])
|
||||
metrics = Enum.zip(metric_headers, metric_values)
|
||||
dimensions = Enum.zip(dimension_headers, dimension_values)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_accounts_for_user(access_token) do
|
||||
url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, property) do
|
||||
url = "#{admin_api_url()}/v1beta/#{property}"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
|
||||
{:error, :not_found}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 property #{property}",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2015-08-14"
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, property, edge) do
|
||||
descending? = edge == :end
|
||||
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: "#{property}",
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "date"}],
|
||||
metrics: [%{name: "screenPageViews"}],
|
||||
orderBys: [
|
||||
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
|
||||
],
|
||||
limit: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["rows"] do
|
||||
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
|
||||
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
|
||||
end
|
||||
@ -1,135 +0,0 @@
|
||||
defmodule Plausible.Google.GA4.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Google Analytics 4 API
|
||||
"""
|
||||
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
:metrics,
|
||||
:date_range,
|
||||
:property,
|
||||
:access_token,
|
||||
:offset,
|
||||
:limit
|
||||
]
|
||||
|
||||
@type t() :: %__MODULE__{
|
||||
dataset: String.t(),
|
||||
dimensions: [String.t()],
|
||||
metrics: [String.t()],
|
||||
date_range: Date.Range.t(),
|
||||
property: term(),
|
||||
access_token: String.t(),
|
||||
offset: non_neg_integer(),
|
||||
limit: non_neg_integer()
|
||||
}
|
||||
|
||||
def full_report do
|
||||
[
|
||||
%__MODULE__{
|
||||
dataset: "imported_visitors",
|
||||
dimensions: ["date"],
|
||||
metrics: [
|
||||
"totalUsers",
|
||||
"screenPageViews",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"sessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_sources",
|
||||
dimensions: [
|
||||
"date",
|
||||
"sessionSource",
|
||||
"sessionMedium",
|
||||
"sessionCampaignName",
|
||||
"sessionManualAdContent",
|
||||
"sessionGoogleAdsKeyword",
|
||||
"pageReferrer"
|
||||
],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_pages",
|
||||
dimensions: ["date", "hostName", "pagePath"],
|
||||
# NOTE: no exits as GA4 DATA API does not provide that metric
|
||||
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_entry_pages",
|
||||
dimensions: ["date", "landingPage"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"userEngagementDuration",
|
||||
"bounces = sessions - engagedSessions"
|
||||
]
|
||||
},
|
||||
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
|
||||
# %__MODULE__{
|
||||
# dataset: "imported_exit_pages",
|
||||
# dimensions: ["date", "ga:exitPagePath"],
|
||||
# metrics: [
|
||||
# "totalUsers",
|
||||
# "sessions",
|
||||
# "screenPageViews",
|
||||
# "userEngagementDuration",
|
||||
# "bounces = sessions - engagedSessions"
|
||||
# ]
|
||||
# },
|
||||
%__MODULE__{
|
||||
dataset: "imported_locations",
|
||||
dimensions: ["date", "countryId", "region", "city"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_devices",
|
||||
dimensions: ["date", "deviceCategory"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_browsers",
|
||||
dimensions: ["date", "browser"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_operating_systems",
|
||||
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
}
|
||||
]
|
||||
end
|
||||
end
|
||||
@ -1,8 +1,4 @@
|
||||
defmodule Plausible.Google.UA.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Universal Analytics API
|
||||
"""
|
||||
|
||||
defmodule Plausible.Google.ReportRequest do
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
@ -1,155 +0,0 @@
|
||||
defmodule Plausible.Google.UA.API do
|
||||
@moduledoc """
|
||||
API for Universal Analytics
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.UA
|
||||
|
||||
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 7_500
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
@spec list_views(access_token :: String.t()) ::
|
||||
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
|
||||
@doc """
|
||||
Lists Google Analytics views grouped by hostname.
|
||||
"""
|
||||
def list_views(access_token) do
|
||||
case UA.HTTP.list_views_for_user(access_token) do
|
||||
{:ok, %{"items" => views}} ->
|
||||
views =
|
||||
views
|
||||
|> Enum.group_by(&view_hostname/1, &view_names/1)
|
||||
|> Enum.sort_by(fn {key, _} -> key end)
|
||||
|
||||
{:ok, views}
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
|
||||
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
|
||||
@doc """
|
||||
Returns a single Google Analytics view if the user has access to it.
|
||||
"""
|
||||
def get_view(access_token, lookup_id) do
|
||||
with {:ok, views} <- list_views(access_token) do
|
||||
views =
|
||||
views
|
||||
|> Enum.map(&elem(&1, 1))
|
||||
|> List.flatten()
|
||||
|
||||
case Enum.find(views, fn {_name, id} -> id == lookup_id end) do
|
||||
{view_name, view_id} ->
|
||||
{:ok, %{id: view_id, name: "#{view_name}"}}
|
||||
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_start_date(access_token, view_id)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_end_date(access_token, view_id)
|
||||
end
|
||||
|
||||
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
|
||||
:ok | {:error, term()}
|
||||
@doc """
|
||||
Imports stats from a Google Analytics UA view to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics reports in batches of #{@per_page} per
|
||||
request. The batches are then passed to persist callback.
|
||||
|
||||
Requests to Google Analytics can fail, and are retried at most
|
||||
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
|
||||
importing has finished or `{:error, term()}` when a request to GA failed too
|
||||
many times.
|
||||
|
||||
Useful links:
|
||||
|
||||
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
|
||||
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
|
||||
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
|
||||
|
||||
"""
|
||||
def import_analytics(date_range, view_id, auth, persist_fn) do
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, view_id, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
|
||||
Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
report_request = %UA.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
view_id: view_id,
|
||||
access_token: access_token,
|
||||
page_token: nil,
|
||||
page_size: @per_page
|
||||
}
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case UA.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, next_page_token}} ->
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
if next_page_token do
|
||||
fetch_and_persist(
|
||||
%UA.ReportRequest{report_request | page_token: next_page_token},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
{:error, cause}
|
||||
else
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp view_hostname(view) do
|
||||
case view do
|
||||
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
|
||||
_any -> "Others"
|
||||
end
|
||||
end
|
||||
|
||||
defp view_names(%{"name" => name, "id" => id}) do
|
||||
{"#{id} - #{name}", id}
|
||||
end
|
||||
end
|
||||
@ -1,191 +0,0 @@
|
||||
defmodule Plausible.Google.UA.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Universal Analytics API.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
@spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
|
||||
{:ok, {[map()], String.t() | nil}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: report_request.view_id,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
|
||||
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
|
||||
pageSize: report_request.page_size,
|
||||
pageToken: report_request.page_token
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
"#{reporting_api_url()}/v4/reports:batchGet",
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
token <- Map.get(report, "nextPageToken"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, token}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, _reason} ->
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"data" => %{} = data,
|
||||
"columnHeader" => %{
|
||||
"dimensions" => dimension_headers,
|
||||
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
|
||||
}
|
||||
}) do
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
rows = Map.get(data, "rows", [])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
|
||||
metrics = Enum.zip(metric_headers, metrics)
|
||||
dimensions = Enum.zip(dimension_headers, dimensions)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_views_for_user(access_token) do
|
||||
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2005-01-01"
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, view_id, edge) do
|
||||
sort_order =
|
||||
if edge == :start do
|
||||
"ASCENDING"
|
||||
else
|
||||
"DESCENDING"
|
||||
end
|
||||
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: view_id,
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "ga:date", histogramBuckets: []}],
|
||||
metrics: [%{expression: "ga:pageviews"}],
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}],
|
||||
pageSize: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v4/reports:batchGet"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["data"]["rows"] do
|
||||
[%{"dimensions" => [date_str]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving UA #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp config, do: Application.get_env(:plausible, :google)
|
||||
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
|
||||
defp api_url, do: Keyword.fetch!(config(), :api_url)
|
||||
end
|
||||
@ -1,276 +0,0 @@
|
||||
defmodule Plausible.Imported.GoogleAnalytics4 do
|
||||
@moduledoc """
|
||||
Import implementation for Google Analytics 4.
|
||||
"""
|
||||
|
||||
use Plausible.Imported.Importer
|
||||
|
||||
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
|
||||
|
||||
@impl true
|
||||
def name(), do: :google_analytics_4
|
||||
|
||||
@impl true
|
||||
def label(), do: "Google Analytics 4"
|
||||
|
||||
@impl true
|
||||
def email_template(), do: "google_analytics_import.html"
|
||||
|
||||
@impl true
|
||||
def parse_args(
|
||||
%{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
|
||||
) do
|
||||
start_date = Date.from_iso8601!(start_date)
|
||||
end_date = Date.from_iso8601!(end_date)
|
||||
date_range = Date.range(start_date, end_date)
|
||||
|
||||
auth = {
|
||||
Map.fetch!(args, "access_token"),
|
||||
Map.fetch!(args, "refresh_token"),
|
||||
Map.fetch!(args, "token_expires_at")
|
||||
}
|
||||
|
||||
[
|
||||
property: property,
|
||||
date_range: date_range,
|
||||
auth: auth
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Imports stats from a Google Analytics 4 property to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics 4 reports which are then passed in batches
|
||||
to Clickhouse by the `Plausible.Imported.Buffer` process.
|
||||
"""
|
||||
@impl true
|
||||
def import_data(site_import, opts) do
|
||||
date_range = Keyword.fetch!(opts, :date_range)
|
||||
property = Keyword.fetch!(opts, :property)
|
||||
auth = Keyword.fetch!(opts, :auth)
|
||||
flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
|
||||
|
||||
{:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
|
||||
|
||||
persist_fn = fn table, rows ->
|
||||
records = from_report(rows, site_import.site_id, site_import.id, table)
|
||||
Plausible.Imported.Buffer.insert_many(buffer, table, records)
|
||||
end
|
||||
|
||||
try do
|
||||
Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
|
||||
after
|
||||
Plausible.Imported.Buffer.flush(buffer)
|
||||
Plausible.Imported.Buffer.stop(buffer)
|
||||
end
|
||||
end
|
||||
|
||||
def from_report(nil, _site_id, _import_id, _metric), do: nil
|
||||
|
||||
def from_report(data, site_id, import_id, table) do
|
||||
Enum.reduce(data, [], fn row, acc ->
|
||||
if Map.get(row.dimensions, "date") in @missing_values do
|
||||
acc
|
||||
else
|
||||
[new_from_report(site_id, import_id, table, row) | acc]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp parse_number(nr) do
|
||||
{float, ""} = Float.parse(nr)
|
||||
round(float)
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_visitors", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_sources", row) do
|
||||
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
|
||||
|
||||
referrer =
|
||||
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
|
||||
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
|
||||
end
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
|
||||
referrer: referrer,
|
||||
# Only `source` exists in GA4 API
|
||||
utm_source: nil,
|
||||
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
|
||||
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
|
||||
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
|
||||
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
|
||||
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# NOTE: no exits metric in GA4 API currently
|
||||
exits: 0,
|
||||
time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
entry_page: row.dimensions |> Map.fetch!("landingPage"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
# NOTE: no exit pages metrics in GA4 API available for now
|
||||
# defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
|
||||
# %{
|
||||
# site_id: site_id,
|
||||
# import_id: import_id,
|
||||
# date: get_date(row),
|
||||
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
|
||||
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
# }
|
||||
# end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_locations", row) do
|
||||
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
|
||||
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
|
||||
city_data = Location.get_city(city_name, country_code)
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
country: country_code,
|
||||
region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
|
||||
city: city_data && city_data.id,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_devices", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@browser_google_to_plausible %{
|
||||
"User-Agent:Opera" => "Opera",
|
||||
"Mozilla Compatible Agent" => "Mobile App",
|
||||
"Android Webview" => "Mobile App",
|
||||
"Android Browser" => "Mobile App",
|
||||
"Safari (in-app)" => "Mobile App",
|
||||
"User-Agent: Mozilla" => "Firefox",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_browsers", row) do
|
||||
browser = Map.fetch!(row.dimensions, "browser")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
browser: Map.get(@browser_google_to_plausible, browser, browser),
|
||||
# Does not exist in GA4 API
|
||||
browser_version: nil,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@os_google_to_plausible %{
|
||||
"Macintosh" => "Mac",
|
||||
"Linux" => "GNU/Linux",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
|
||||
os = Map.fetch!(row.dimensions, "operatingSystem")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
operating_system: Map.get(@os_google_to_plausible, os, os),
|
||||
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp get_date(%{dimensions: %{"date" => date}}) do
|
||||
date
|
||||
|> Timex.parse!("%Y%m%d", :strftime)
|
||||
|> NaiveDateTime.to_date()
|
||||
end
|
||||
|
||||
defp default_if_missing(value, default \\ nil)
|
||||
defp default_if_missing(value, default) when value in @missing_values, do: default
|
||||
defp default_if_missing(value, _default), do: value
|
||||
|
||||
defp parse_referrer(nil), do: nil
|
||||
defp parse_referrer("(direct)"), do: nil
|
||||
defp parse_referrer("google"), do: "Google"
|
||||
defp parse_referrer("bing"), do: "Bing"
|
||||
defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
|
||||
|
||||
defp parse_referrer(ref) do
|
||||
RefInspector.parse("https://" <> ref)
|
||||
|> PlausibleWeb.RefInspector.parse()
|
||||
end
|
||||
end
|
||||
@ -1,85 +0,0 @@
|
||||
defmodule CompiledRegex do
|
||||
@moduledoc """
|
||||
Ensures that the regex is compiled on load
|
||||
"""
|
||||
use Ecto.Type
|
||||
|
||||
def type, do: :string
|
||||
|
||||
def cast(val) when is_binary(val), do: {:ok, val}
|
||||
def cast(_), do: :error
|
||||
|
||||
def load(val), do: {:ok, Regex.compile!(val)}
|
||||
def dump(val), do: {:ok, val}
|
||||
end
|
||||
|
||||
defmodule Plausible.Shield.PageRule do
|
||||
@moduledoc """
|
||||
Schema for Pages block list
|
||||
"""
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
@type t() :: %__MODULE__{}
|
||||
|
||||
@primary_key {:id, :binary_id, autogenerate: true}
|
||||
schema "shield_rules_page" do
|
||||
belongs_to :site, Plausible.Site
|
||||
field :page_path, :string
|
||||
field :page_path_pattern, CompiledRegex
|
||||
field :action, Ecto.Enum, values: [:deny, :allow], default: :deny
|
||||
field :added_by, :string
|
||||
|
||||
# If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache`
|
||||
field :from_cache?, :boolean, virtual: true, default: false
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def changeset(rule \\ %__MODULE__{}, attrs) do
|
||||
rule
|
||||
|> cast(attrs, [:site_id, :page_path])
|
||||
|> validate_required([:site_id, :page_path])
|
||||
|> validate_length(:page_path, max: 250)
|
||||
|> validate_change(:page_path, fn :page_path, p ->
|
||||
if not String.starts_with?(p, "/") do
|
||||
[page_path: "must start with /"]
|
||||
else
|
||||
[]
|
||||
end
|
||||
end)
|
||||
|> store_regex()
|
||||
|> unique_constraint(:page_path_pattern,
|
||||
name: :shield_rules_page_site_id_page_path_pattern_index,
|
||||
error_key: :page_path,
|
||||
message: "rule already exists"
|
||||
)
|
||||
end
|
||||
|
||||
defp store_regex(changeset) do
|
||||
case get_field(changeset, :page_path) do
|
||||
"/" <> _ = page_path ->
|
||||
regex =
|
||||
page_path
|
||||
|> Regex.escape()
|
||||
|> String.replace("\\*\\*", ".*")
|
||||
|> String.replace("\\*", ".*")
|
||||
|
||||
regex = "^#{regex}$"
|
||||
|
||||
verify_valid_regex(changeset, regex)
|
||||
|
||||
_ ->
|
||||
changeset
|
||||
end
|
||||
end
|
||||
|
||||
defp verify_valid_regex(changeset, regex) do
|
||||
case Regex.compile(regex) do
|
||||
{:ok, _} ->
|
||||
put_change(changeset, :page_path_pattern, regex)
|
||||
|
||||
{:error, _} ->
|
||||
add_error(changeset, :page_path, "could not compile regular expression")
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,65 +0,0 @@
|
||||
defmodule Plausible.Shield.PageRuleCache do
|
||||
@moduledoc """
|
||||
Allows retrieving Page Rules by domain
|
||||
"""
|
||||
alias Plausible.Shield.PageRule
|
||||
|
||||
import Ecto.Query
|
||||
use Plausible.Cache
|
||||
|
||||
@cache_name :page_blocklist_by_domain
|
||||
|
||||
@cached_schema_fields ~w(
|
||||
id
|
||||
page_path_pattern
|
||||
action
|
||||
)a
|
||||
|
||||
@impl true
|
||||
def name(), do: @cache_name
|
||||
|
||||
@impl true
|
||||
def child_id(), do: :cache_page_blocklist
|
||||
|
||||
@impl true
|
||||
def count_all() do
|
||||
Plausible.Repo.aggregate(PageRule, :count)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def base_db_query() do
|
||||
from rule in PageRule,
|
||||
inner_join: s in assoc(rule, :site),
|
||||
select: {
|
||||
s.domain,
|
||||
s.domain_changed_from,
|
||||
%{struct(rule, ^@cached_schema_fields) | from_cache?: true}
|
||||
}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def get_from_source(domain) do
|
||||
query =
|
||||
base_db_query()
|
||||
|> where([..., site], site.domain == ^domain)
|
||||
|
||||
case Plausible.Repo.one(query) do
|
||||
{_, _, rule} -> %PageRule{rule | from_cache?: false}
|
||||
_any -> nil
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def unwrap_cache_keys(items) do
|
||||
Enum.reduce(items, [], fn
|
||||
{domain, nil, object}, acc ->
|
||||
[{domain, object} | acc]
|
||||
|
||||
{domain, domain_changed_from, object}, acc ->
|
||||
[
|
||||
{domain, object},
|
||||
{domain_changed_from, object} | acc
|
||||
]
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -1,136 +0,0 @@
|
||||
defmodule Plausible.Shields.CountryTest do
|
||||
use Plausible.DataCase
|
||||
import Plausible.Shields
|
||||
|
||||
setup do
|
||||
site = insert(:site)
|
||||
{:ok, %{site: site}}
|
||||
end
|
||||
|
||||
describe "add_country_rule/2" do
|
||||
test "no input", %{site: site} do
|
||||
assert {:error, changeset} = add_country_rule(site, %{})
|
||||
assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}]
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "unsupported country", %{site: site} do
|
||||
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"})
|
||||
assert changeset.errors == [country_code: {"is invalid", []}]
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "incorrect country format", %{site: site} do
|
||||
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"})
|
||||
|
||||
assert changeset.errors ==
|
||||
[
|
||||
{:country_code, {"is invalid", []}},
|
||||
{:country_code,
|
||||
{"should be %{count} character(s)",
|
||||
[count: 2, validation: :length, kind: :is, type: :string]}}
|
||||
]
|
||||
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "double insert", %{site: site} do
|
||||
assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
|
||||
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"})
|
||||
refute changeset.valid?
|
||||
|
||||
assert changeset.errors == [
|
||||
country_code:
|
||||
{"has already been taken",
|
||||
[
|
||||
{:constraint, :unique},
|
||||
{:constraint_name, "shield_rules_country_site_id_country_code_index"}
|
||||
]}
|
||||
]
|
||||
end
|
||||
|
||||
test "over limit", %{site: site} do
|
||||
country_codes =
|
||||
Location.Country.all()
|
||||
|> Enum.take(Plausible.Shields.maximum_country_rules())
|
||||
|> Enum.map(& &1.alpha_2)
|
||||
|
||||
for cc <- country_codes do
|
||||
assert {:ok, _} =
|
||||
add_country_rule(site, %{"country_code" => cc})
|
||||
end
|
||||
|
||||
assert count_country_rules(site) == maximum_country_rules()
|
||||
|
||||
assert {:error, changeset} =
|
||||
add_country_rule(site, %{"country_code" => "US"})
|
||||
|
||||
refute changeset.valid?
|
||||
assert changeset.errors == [country_code: {"maximum reached", []}]
|
||||
end
|
||||
|
||||
test "with added_by", %{site: site} do
|
||||
assert {:ok, rule} =
|
||||
add_country_rule(site, %{"country_code" => "EE"},
|
||||
added_by: build(:user, name: "Joe", email: "joe@example.com")
|
||||
)
|
||||
|
||||
assert rule.added_by == "Joe <joe@example.com>"
|
||||
end
|
||||
end
|
||||
|
||||
describe "remove_country_rule/2" do
|
||||
test "is idempontent", %{site: site} do
|
||||
{:ok, rule} = add_country_rule(site, %{"country_code" => "EE"})
|
||||
assert remove_country_rule(site, rule.id) == :ok
|
||||
refute Repo.get(Plausible.Shield.CountryRule, rule.id)
|
||||
assert remove_country_rule(site, rule.id) == :ok
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_country_rules/1" do
|
||||
test "empty", %{site: site} do
|
||||
assert(list_country_rules(site) == [])
|
||||
end
|
||||
|
||||
@tag :slow
|
||||
test "many", %{site: site} do
|
||||
{:ok, r1} = add_country_rule(site, %{"country_code" => "EE"})
|
||||
:timer.sleep(1000)
|
||||
{:ok, r2} = add_country_rule(site, %{"country_code" => "PL"})
|
||||
assert [^r2, ^r1] = list_country_rules(site)
|
||||
end
|
||||
end
|
||||
|
||||
describe "count_country_rules/1" do
|
||||
test "counts", %{site: site} do
|
||||
assert count_country_rules(site) == 0
|
||||
{:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
|
||||
assert count_country_rules(site) == 1
|
||||
{:ok, _} = add_country_rule(site, %{"country_code" => "PL"})
|
||||
assert count_country_rules(site) == 2
|
||||
end
|
||||
end
|
||||
|
||||
describe "Country Rules" do
|
||||
test "end to end", %{site: site} do
|
||||
site2 = insert(:site)
|
||||
|
||||
assert count_country_rules(site.id) == 0
|
||||
assert list_country_rules(site.id) == []
|
||||
|
||||
assert {:ok, rule} =
|
||||
add_country_rule(site.id, %{"country_code" => "EE"})
|
||||
|
||||
add_country_rule(site2, %{"country_code" => "PL"})
|
||||
|
||||
assert count_country_rules(site) == 1
|
||||
assert [^rule] = list_country_rules(site)
|
||||
assert rule.country_code == "EE"
|
||||
assert rule.action == :deny
|
||||
refute rule.from_cache?
|
||||
assert country_blocked?(site, "ee")
|
||||
refute country_blocked?(site, "xx")
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,147 +0,0 @@
|
||||
defmodule Plausible.Shields.IPTest do
|
||||
use Plausible.DataCase
|
||||
import Plausible.Shields
|
||||
|
||||
setup do
|
||||
site = insert(:site)
|
||||
{:ok, %{site: site}}
|
||||
end
|
||||
|
||||
describe "add_ip_rule/2" do
|
||||
test "no input", %{site: site} do
|
||||
assert {:error, changeset} = add_ip_rule(site, %{})
|
||||
assert changeset.errors == [inet: {"can't be blank", [validation: :required]}]
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "unsupported netmask", %{site: site} do
|
||||
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"})
|
||||
assert changeset.errors == [inet: {"netmask unsupported", []}]
|
||||
refute changeset.valid?
|
||||
assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"})
|
||||
end
|
||||
|
||||
test "incorrect ip", %{site: site} do
|
||||
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"})
|
||||
|
||||
assert changeset.errors == [
|
||||
inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]}
|
||||
]
|
||||
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "non-strict IPs", %{site: site} do
|
||||
assert {:error, _} = add_ip_rule(site, %{"inet" => "111"})
|
||||
end
|
||||
|
||||
test "double insert", %{site: site} do
|
||||
assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
|
||||
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
|
||||
refute changeset.valid?
|
||||
|
||||
assert changeset.errors == [
|
||||
inet:
|
||||
{"has already been taken",
|
||||
[
|
||||
{:constraint, :unique},
|
||||
{:constraint_name, "shield_rules_ip_site_id_inet_index"}
|
||||
]}
|
||||
]
|
||||
end
|
||||
|
||||
test "ipv6", %{site: site} do
|
||||
assert {:ok, rule} =
|
||||
add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"})
|
||||
|
||||
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
|
||||
end
|
||||
|
||||
test "ipv4", %{site: site} do
|
||||
assert {:ok, rule} =
|
||||
add_ip_rule(site, %{"inet" => "1.1.1.1"})
|
||||
|
||||
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
|
||||
end
|
||||
|
||||
test "over limit", %{site: site} do
|
||||
for i <- 1..maximum_ip_rules() do
|
||||
assert {:ok, _} =
|
||||
add_ip_rule(site, %{"inet" => "1.1.1.#{i}"})
|
||||
end
|
||||
|
||||
assert count_ip_rules(site) == maximum_ip_rules()
|
||||
|
||||
assert {:error, changeset} =
|
||||
add_ip_rule(site, %{"inet" => "1.1.1.31"})
|
||||
|
||||
refute changeset.valid?
|
||||
assert changeset.errors == [inet: {"maximum reached", []}]
|
||||
end
|
||||
|
||||
test "with added_by", %{site: site} do
|
||||
assert {:ok, rule} =
|
||||
add_ip_rule(site, %{"inet" => "1.1.1.1"},
|
||||
added_by: build(:user, name: "Joe", email: "joe@example.com")
|
||||
)
|
||||
|
||||
assert rule.added_by == "Joe <joe@example.com>"
|
||||
end
|
||||
end
|
||||
|
||||
describe "remove_ip_rule/2" do
|
||||
test "is idempontent", %{site: site} do
|
||||
{:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
|
||||
assert remove_ip_rule(site, rule.id) == :ok
|
||||
refute Repo.get(Plausible.Shield.IPRule, rule.id)
|
||||
assert remove_ip_rule(site, rule.id) == :ok
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_ip_rules/1" do
|
||||
test "empty", %{site: site} do
|
||||
assert(list_ip_rules(site) == [])
|
||||
end
|
||||
|
||||
@tag :slow
|
||||
test "many", %{site: site} do
|
||||
{:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
|
||||
:timer.sleep(1000)
|
||||
{:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
|
||||
assert [^r2, ^r1] = list_ip_rules(site)
|
||||
end
|
||||
end
|
||||
|
||||
describe "count_ip_rules/1" do
|
||||
test "counts", %{site: site} do
|
||||
assert count_ip_rules(site) == 0
|
||||
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
|
||||
assert count_ip_rules(site) == 1
|
||||
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
|
||||
assert count_ip_rules(site) == 2
|
||||
end
|
||||
end
|
||||
|
||||
describe "IP Rules" do
|
||||
test "end to end", %{site: site} do
|
||||
site2 = insert(:site)
|
||||
|
||||
assert count_ip_rules(site.id) == 0
|
||||
assert list_ip_rules(site.id) == []
|
||||
|
||||
assert {:ok, rule} =
|
||||
add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"})
|
||||
|
||||
add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"})
|
||||
|
||||
assert count_ip_rules(site) == 1
|
||||
assert [^rule] = list_ip_rules(site)
|
||||
assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32}
|
||||
assert rule.description == "Localhost"
|
||||
assert rule.action == :deny
|
||||
refute rule.from_cache?
|
||||
assert ip_blocked?(site, "127.0.0.1")
|
||||
refute ip_blocked?(site, "127.0.0.2")
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,174 +0,0 @@
|
||||
defmodule Plausible.Shields.PageTest do
|
||||
use Plausible.DataCase
|
||||
import Plausible.Shields
|
||||
|
||||
setup do
|
||||
site = insert(:site)
|
||||
{:ok, %{site: site}}
|
||||
end
|
||||
|
||||
describe "add_page_rule/2" do
|
||||
test "no input", %{site: site} do
|
||||
assert {:error, changeset} = add_page_rule(site, %{})
|
||||
assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}]
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "no slash", %{site: site} do
|
||||
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"})
|
||||
assert changeset.errors == [page_path: {"must start with /", []}]
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "lengthy", %{site: site} do
|
||||
long = "/" <> :binary.copy("a", 251)
|
||||
assert {:error, changeset} = add_page_rule(site, %{"page_path" => long})
|
||||
assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors
|
||||
refute changeset.valid?
|
||||
end
|
||||
|
||||
test "double insert", %{site: site} do
|
||||
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
refute changeset.valid?
|
||||
|
||||
assert changeset.errors == [
|
||||
page_path:
|
||||
{"rule already exists",
|
||||
[
|
||||
{:constraint, :unique},
|
||||
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
|
||||
]}
|
||||
]
|
||||
end
|
||||
|
||||
test "equivalent rules are counted as dupes", %{site: site} do
|
||||
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
|
||||
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"})
|
||||
|
||||
assert changeset.errors == [
|
||||
page_path:
|
||||
{"rule already exists",
|
||||
[
|
||||
{:constraint, :unique},
|
||||
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
|
||||
]}
|
||||
]
|
||||
end
|
||||
|
||||
test "regex storage: wildcard", %{site: site} do
|
||||
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"})
|
||||
assert rule.page_path_pattern == "^/test/.*$"
|
||||
end
|
||||
|
||||
test "regex storage: no wildcard", %{site: site} do
|
||||
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
assert rule.page_path_pattern == "^/test$"
|
||||
end
|
||||
|
||||
test "regex storage: escaping", %{site: site} do
|
||||
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"})
|
||||
assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$"
|
||||
end
|
||||
|
||||
test "over limit", %{site: site} do
|
||||
for i <- 1..maximum_page_rules() do
|
||||
assert {:ok, _} =
|
||||
add_page_rule(site, %{"page_path" => "/test/#{i}"})
|
||||
end
|
||||
|
||||
assert count_page_rules(site) == maximum_page_rules()
|
||||
|
||||
assert {:error, changeset} =
|
||||
add_page_rule(site, %{"page_path" => "/test/31"})
|
||||
|
||||
refute changeset.valid?
|
||||
assert changeset.errors == [page_path: {"maximum reached", []}]
|
||||
end
|
||||
|
||||
test "with added_by", %{site: site} do
|
||||
assert {:ok, rule} =
|
||||
add_page_rule(site, %{"page_path" => "/test"},
|
||||
added_by: build(:user, name: "Joe", email: "joe@example.com")
|
||||
)
|
||||
|
||||
assert rule.added_by == "Joe <joe@example.com>"
|
||||
end
|
||||
end
|
||||
|
||||
describe "page pattern matching" do
|
||||
test "no wildcard", %{site: site} do
|
||||
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
assert page_blocked?(site, "/test")
|
||||
refute page_blocked?(site, "/test/hello")
|
||||
refute page_blocked?(site, "test")
|
||||
end
|
||||
|
||||
test "wildcard", %{site: site} do
|
||||
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
|
||||
refute page_blocked?(site, "/test")
|
||||
assert page_blocked?(site, "/test/")
|
||||
assert page_blocked?(site, "/test/hello")
|
||||
refute page_blocked?(site, "test")
|
||||
refute page_blocked?(site, "/testing")
|
||||
end
|
||||
end
|
||||
|
||||
describe "remove_page_rule/2" do
|
||||
test "is idempontent", %{site: site} do
|
||||
{:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
assert remove_page_rule(site, rule.id) == :ok
|
||||
refute Repo.get(Plausible.Shield.PageRule, rule.id)
|
||||
assert remove_page_rule(site, rule.id) == :ok
|
||||
end
|
||||
end
|
||||
|
||||
describe "list_page_rules/1" do
|
||||
test "empty", %{site: site} do
|
||||
assert(list_page_rules(site) == [])
|
||||
end
|
||||
|
||||
@tag :slow
|
||||
test "many", %{site: site} do
|
||||
{:ok, r1} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
:timer.sleep(1000)
|
||||
{:ok, r2} = add_page_rule(site, %{"page_path" => "/test"})
|
||||
assert [^r2, ^r1] = list_page_rules(site)
|
||||
end
|
||||
end
|
||||
|
||||
describe "count_page_rules/1" do
|
||||
test "counts", %{site: site} do
|
||||
assert count_page_rules(site) == 0
|
||||
{:ok, _} = add_page_rule(site, %{"page_path" => "/test1"})
|
||||
assert count_page_rules(site) == 1
|
||||
{:ok, _} = add_page_rule(site, %{"page_path" => "/test2"})
|
||||
assert count_page_rules(site) == 2
|
||||
end
|
||||
end
|
||||
|
||||
describe "Page Rules" do
|
||||
test "end to end", %{site: site} do
|
||||
site2 = insert(:site)
|
||||
|
||||
assert count_page_rules(site.id) == 0
|
||||
assert list_page_rules(site.id) == []
|
||||
|
||||
assert {:ok, rule} =
|
||||
add_page_rule(site.id, %{
|
||||
"page_path" => "/test"
|
||||
})
|
||||
|
||||
add_page_rule(site2, %{"page_path" => "/test"})
|
||||
|
||||
assert count_page_rules(site) == 1
|
||||
assert [%{id: rule_id}] = list_page_rules(site)
|
||||
assert rule.id == rule_id
|
||||
assert rule.page_path == "/test"
|
||||
assert rule.action == :deny
|
||||
refute rule.from_cache?
|
||||
assert page_blocked?(site, "/test")
|
||||
refute page_blocked?(site, "/testing")
|
||||
end
|
||||
end
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue