Compare commits
117 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
a7603c9e49 | 3 years ago |
|
|
cffff0340c | 3 years ago |
|
|
33eed9d7db | 3 years ago |
|
|
e5b7f1afd0 | 3 years ago |
|
|
f966419a4a | 3 years ago |
|
|
f9f0407d68 | 3 years ago |
|
|
e5b56dbe62 | 3 years ago |
|
|
3115c6e7a8 | 3 years ago |
|
|
6af80dd246 | 3 years ago |
|
|
e6d83e946f | 3 years ago |
|
|
bd73fc8266 | 3 years ago |
|
|
2ba4988c95 | 3 years ago |
|
|
1f778e0c11 | 3 years ago |
|
|
9f27fa303c | 3 years ago |
|
|
23a3699dd7 | 3 years ago |
|
|
71fe541359 | 3 years ago |
|
|
5bf59d1d8a | 3 years ago |
|
|
c263df5805 | 3 years ago |
|
|
b31433a7bf | 3 years ago |
|
|
edf70d14b6 | 3 years ago |
|
|
7523abe93e | 3 years ago |
|
|
604bf88451 | 3 years ago |
|
|
4af7019011 | 3 years ago |
|
|
2fae0146a4 | 3 years ago |
|
|
9989ce6927 | 3 years ago |
|
|
53f94a9f82 | 3 years ago |
|
|
ba5b80a8c0 | 3 years ago |
|
|
1d017e86a1 | 3 years ago |
|
|
52c226c428 | 3 years ago |
|
|
fb61f0b425 | 3 years ago |
|
|
d6e1e8bebd | 3 years ago |
|
|
561dcd821e | 3 years ago |
|
|
fd879eeb16 | 3 years ago |
|
|
8992c8ee07 | 3 years ago |
|
|
c32779a3e5 | 3 years ago |
|
|
d6e81670e4 | 3 years ago |
|
|
5f9465614b | 3 years ago |
|
|
32ab138301 | 3 years ago |
|
|
c219652dae | 3 years ago |
|
|
253fb5d67d | 3 years ago |
|
|
1909743b90 | 3 years ago |
|
|
64f35bf8f6 | 3 years ago |
|
|
279e89c693 | 3 years ago |
|
|
4242b52be4 | 3 years ago |
|
|
dfcc8d794a | 3 years ago |
|
|
02d2256483 | 3 years ago |
|
|
07b714a143 | 3 years ago |
|
|
c08aa29ad8 | 3 years ago |
|
|
0a0c7ce165 | 3 years ago |
|
|
8c0e94f72c | 3 years ago |
|
|
4ae00540dc | 3 years ago |
|
|
5e74b1cf74 | 3 years ago |
|
|
d3586a81e3 | 3 years ago |
|
|
59afa20955 | 3 years ago |
|
|
e8f3946dde | 3 years ago |
|
|
4d7d88cfec | 3 years ago |
|
|
f2350b5165 | 3 years ago |
|
|
5a3072ca21 | 3 years ago |
|
|
6eef32a8ff | 3 years ago |
|
|
7641c66a2b | 3 years ago |
|
|
dea9287d74 | 3 years ago |
|
|
c5881cdc6d | 3 years ago |
|
|
ca7edd47cb | 3 years ago |
|
|
437a3350ff | 3 years ago |
|
|
bb8a272a36 | 3 years ago |
|
|
b68e3aa6a2 | 3 years ago |
|
|
83a46fb47e | 3 years ago |
|
|
11368e5143 | 3 years ago |
|
|
a9d3c03782 | 3 years ago |
|
|
26d41ddbb9 | 3 years ago |
|
|
0cdba7d407 | 3 years ago |
|
|
c6d98397a8 | 3 years ago |
|
|
c60a2faee4 | 3 years ago |
|
|
70d21f1b69 | 3 years ago |
|
|
8d977e0f76 | 3 years ago |
|
|
fecd665c19 | 3 years ago |
|
|
d5048fd6b4 | 3 years ago |
|
|
5eb9d724e5 | 3 years ago |
|
|
ea38b45685 | 3 years ago |
|
|
39aa81a16f | 3 years ago |
|
|
fdbe4cc0a4 | 3 years ago |
|
|
0fe5682f91 | 3 years ago |
|
|
316148e788 | 3 years ago |
|
|
f3423aefec | 3 years ago |
|
|
31cf3e54f8 | 3 years ago |
|
|
f1b6a672d4 | 3 years ago |
|
|
518cdb3307 | 3 years ago |
|
|
b7b5dcf4c9 | 3 years ago |
|
|
8a8df685a7 | 3 years ago |
|
|
dd428430f5 | 3 years ago |
|
|
cc867396b2 | 3 years ago |
|
|
bea8ceeffb | 3 years ago |
|
|
c822f8de1e | 3 years ago |
|
|
c6e06359c6 | 3 years ago |
|
|
19ce62db61 | 3 years ago |
|
|
a1fedb751b | 3 years ago |
|
|
0f0f604d2e | 3 years ago |
|
|
bdaf80c05d | 3 years ago |
|
|
acd4f33c8e | 3 years ago |
|
|
bca617d0bf | 3 years ago |
|
|
52f584efa9 | 3 years ago |
|
|
8ba2b934b3 | 3 years ago |
|
|
d74b1d5e60 | 3 years ago |
|
|
6035618213 | 3 years ago |
|
|
23b9032148 | 3 years ago |
|
|
5cdca6f408 | 3 years ago |
|
|
eceac8afd5 | 3 years ago |
|
|
e96a52c588 | 3 years ago |
|
|
8e8790dd30 | 3 years ago |
|
|
c430274d7e | 3 years ago |
|
|
a6b1a6ebc7 | 3 years ago |
|
|
926de4dd10 | 3 years ago |
|
|
672d682e95 | 3 years ago |
|
|
0065cd3052 | 3 years ago |
|
|
ac7da6a9d4 | 3 years ago |
|
|
423f72a0ad | 3 years ago |
|
|
f8b4d5066a | 3 years ago |
@ -0,0 +1,73 @@
|
||||
# This file excludes paths from the Docker build context.
|
||||
#
|
||||
# By default, Docker's build context includes all files (and folders) in the
|
||||
# current directory. Even if a file isn't copied into the container it is still sent to
|
||||
# the Docker daemon.
|
||||
#
|
||||
# There are multiple reasons to exclude files from the build context:
|
||||
#
|
||||
# 1. Prevent nested folders from being copied into the container (ex: exclude
|
||||
# /assets/node_modules when copying /assets)
|
||||
# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc)
|
||||
# 3. Avoid sending files containing sensitive information
|
||||
#
|
||||
# More information on using .dockerignore is available here:
|
||||
# https://docs.docker.com/engine/reference/builder/#dockerignore-file
|
||||
|
||||
.dockerignore
|
||||
|
||||
# Ignore git, but keep git HEAD and refs to access current commit hash if needed:
|
||||
#
|
||||
# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat
|
||||
# d0b8727759e1e0e7aa3d41707d12376e373d5ecc
|
||||
.git
|
||||
!.git/HEAD
|
||||
!.git/refs
|
||||
|
||||
# Common development/test artifacts
|
||||
/cover/
|
||||
/doc/
|
||||
/test/
|
||||
/tmp/
|
||||
.elixir_ls
|
||||
plausible-report.xml
|
||||
.env
|
||||
.idea
|
||||
*.iml
|
||||
*.log
|
||||
*.code-workspace
|
||||
.vscode
|
||||
|
||||
# Mix artifacts
|
||||
/_build/
|
||||
/deps/
|
||||
*.ez
|
||||
|
||||
# Generated on crash by the VM
|
||||
erl_crash.dump
|
||||
|
||||
# If NPM crashes, it generates a log, let's ignore it too.
|
||||
npm-debug.log
|
||||
|
||||
# Static artifacts - These should be fetched and built inside the Docker image
|
||||
/assets/node_modules/
|
||||
/tracker/node_modules/
|
||||
/priv/static/cache_manifest.json
|
||||
/priv/static/css
|
||||
/priv/static/js
|
||||
/priv/version.json
|
||||
|
||||
# Auto-generated tracker files
|
||||
/priv/tracker/js/*.js
|
||||
|
||||
# Dializer
|
||||
/priv/plts/*.plt
|
||||
/priv/plts/*.plt.hash
|
||||
|
||||
# Geolocation databases
|
||||
/priv/geodb/*.mmdb
|
||||
/priv/geodb/*.mmdb.gz
|
||||
|
||||
# Docker volumes
|
||||
.clickhouse_db_vol*
|
||||
plausible_db*
|
||||
@ -0,0 +1,67 @@
|
||||
name: Build Private Images GHCR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, stable]
|
||||
tags: ['r*']
|
||||
pull_request:
|
||||
types: [synchronize, labeled]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') }}
|
||||
runs-on: buildjet-16vcpu-ubuntu-2204
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5.0.0
|
||||
with:
|
||||
images: ghcr.io/plausible/analytics/ee
|
||||
tags: |
|
||||
type=ref,event=pr
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
MIX_ENV=prod
|
||||
BUILD_METADATA=${{ steps.meta.outputs.json }}
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Notify team on failure
|
||||
if: ${{ failure() }}
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
|
||||
method: 'POST'
|
||||
customHeaders: '{"Content-Type": "application/json"}'
|
||||
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-private-images.yml\">Build failed</a>"}'
|
||||
@ -0,0 +1,68 @@
|
||||
name: Build Public Images GHCR
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: buildjet-16vcpu-ubuntu-2204
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/plausible/community-edition
|
||||
tags: |
|
||||
type=semver,pattern={{version}},prefix=v
|
||||
type=semver,pattern={{major}}.{{minor}},prefix=v
|
||||
type=semver,pattern={{major}},prefix=v
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64, linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
MIX_ENV=small
|
||||
BUILD_METADATA=${{ steps.meta.outputs.json }}
|
||||
ERL_FLAGS=+JPperf true
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Notify team on failure
|
||||
if: ${{ failure() }}
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
|
||||
method: 'POST'
|
||||
customHeaders: '{"Content-Type": "application/json"}'
|
||||
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-public-images.yml\">Build failed</a>"}'
|
||||
@ -0,0 +1,28 @@
|
||||
name: Validate migrations
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'priv/repo/migrations/**'
|
||||
- 'priv/ingest_repo/migrations/**'
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: App code does not change at the same time
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: changes
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
lib:
|
||||
- 'lib/**'
|
||||
- 'extra/**'
|
||||
- 'config/**'
|
||||
|
||||
- if: steps.changes.outputs.lib == 'true'
|
||||
run: |
|
||||
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
|
||||
exit 1
|
||||
@ -0,0 +1,5 @@
|
||||
Plausible copyright notice
|
||||
|
||||
This software is Copyright 2024 Plausible Insights OÜ
|
||||
|
||||
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io
|
||||
@ -0,0 +1,247 @@
|
||||
import React from 'react';
|
||||
import { withRouter } from 'react-router-dom'
|
||||
import Chart from 'chart.js/auto';
|
||||
import { navigateToQuery } from '../../query'
|
||||
import GraphTooltip from './graph-tooltip'
|
||||
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
|
||||
import dateFormatter from './date-formatter';
|
||||
import FadeIn from '../../fade-in';
|
||||
import classNames from 'classnames';
|
||||
|
||||
const calculateMaximumY = function(dataset) {
|
||||
const yAxisValues = dataset
|
||||
.flatMap((item) => item.data)
|
||||
.map((item) => item || 0)
|
||||
|
||||
if (yAxisValues) {
|
||||
return Math.max(...yAxisValues)
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
class LineGraph extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.regenerateChart = this.regenerateChart.bind(this);
|
||||
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
|
||||
}
|
||||
|
||||
getGraphMetric() {
|
||||
let metric = this.props.graphData.metric
|
||||
|
||||
if (metric == 'visitors' && this.props.query.filters.goal) {
|
||||
return 'conversions'
|
||||
} else {
|
||||
return metric
|
||||
}
|
||||
}
|
||||
|
||||
regenerateChart() {
|
||||
const { graphData, query } = this.props
|
||||
const metric = this.getGraphMetric()
|
||||
const graphEl = document.getElementById("main-graph-canvas")
|
||||
this.ctx = graphEl.getContext('2d');
|
||||
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
|
||||
|
||||
return new Chart(this.ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: graphData.labels,
|
||||
datasets: dataSet
|
||||
},
|
||||
options: {
|
||||
animation: false,
|
||||
plugins: {
|
||||
legend: { display: false },
|
||||
tooltip: {
|
||||
enabled: false,
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
position: 'average',
|
||||
external: GraphTooltip(graphData, metric, query)
|
||||
},
|
||||
},
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
onResize: this.updateWindowDimensions,
|
||||
elements: { line: { tension: 0 }, point: { radius: 0 } },
|
||||
onClick: this.maybeHopToHoveredPeriod.bind(this),
|
||||
scale: {
|
||||
ticks: { precision: 0, maxTicksLimit: 8 }
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
ticks: {
|
||||
callback: METRIC_FORMATTER[metric],
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
},
|
||||
grid: {
|
||||
zeroLineColor: 'transparent',
|
||||
drawBorder: false,
|
||||
}
|
||||
},
|
||||
yComparison: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
display: false,
|
||||
grid: { display: false },
|
||||
},
|
||||
x: {
|
||||
grid: { display: false },
|
||||
ticks: {
|
||||
callback: function(val, _index, _ticks) {
|
||||
if (this.getLabelForValue(val) == "__blank__") return ""
|
||||
|
||||
const hasMultipleYears =
|
||||
graphData.labels
|
||||
.filter((date) => typeof date === 'string')
|
||||
.map(date => date.split('-')[0])
|
||||
.filter((value, index, list) => list.indexOf(value) === index)
|
||||
.length > 1
|
||||
|
||||
if (graphData.interval === 'hour' && query.period !== 'day') {
|
||||
const date = dateFormatter({
|
||||
interval: "date",
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
const hour = dateFormatter({
|
||||
interval: graphData.interval,
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
// Returns a combination of date and hour. This is because
|
||||
// small intervals like hour may return multiple days
|
||||
// depending on the query period.
|
||||
return `${date}, ${hour}`
|
||||
}
|
||||
|
||||
if (graphData.interval === 'minute' && query.period !== 'realtime') {
|
||||
return dateFormatter({
|
||||
interval: "hour", longForm: false, period: query.period,
|
||||
})(this.getLabelForValue(val))
|
||||
}
|
||||
|
||||
return dateFormatter({
|
||||
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
},
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
}
|
||||
}
|
||||
},
|
||||
interaction: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
repositionTooltip(e) {
|
||||
const tooltipEl = document.getElementById('chartjs-tooltip');
|
||||
if (tooltipEl && window.innerWidth >= 768) {
|
||||
if (e.clientX > 0.66 * window.innerWidth) {
|
||||
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
|
||||
tooltipEl.style.left = null;
|
||||
} else {
|
||||
tooltipEl.style.right = null;
|
||||
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
|
||||
}
|
||||
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
|
||||
tooltipEl.style.opacity = 1;
|
||||
}
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
if (this.props.graphData) {
|
||||
this.chart = this.regenerateChart();
|
||||
}
|
||||
window.addEventListener('mousemove', this.repositionTooltip);
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps) {
|
||||
const { graphData, darkTheme } = this.props;
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
|
||||
if (
|
||||
graphData !== prevProps.graphData ||
|
||||
darkTheme !== prevProps.darkTheme
|
||||
) {
|
||||
|
||||
if (graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
this.chart = this.regenerateChart();
|
||||
this.chart.update();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (!graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
// Ensure that the tooltip doesn't hang around when we are loading more data
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
if (tooltip) {
|
||||
tooltip.style.opacity = 0;
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
window.removeEventListener('mousemove', this.repositionTooltip)
|
||||
}
|
||||
|
||||
/**
|
||||
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
|
||||
* @param {*} chart - The chart instance.
|
||||
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
|
||||
*/
|
||||
updateWindowDimensions(chart, dimensions) {
|
||||
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
|
||||
}
|
||||
|
||||
maybeHopToHoveredPeriod(e) {
|
||||
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
|
||||
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
|
||||
|
||||
if (this.props.graphData.interval === 'month') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
|
||||
} else if (this.props.graphData.interval === 'date') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
const { graphData } = this.props
|
||||
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
|
||||
|
||||
return (
|
||||
<FadeIn show={graphData}>
|
||||
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
|
||||
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
|
||||
</div>
|
||||
</FadeIn>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default withRouter(LineGraph)
|
||||
@ -0,0 +1,17 @@
|
||||
import React from "react"
|
||||
|
||||
export default function SamplingNotice({topStatData}) {
|
||||
const samplePercent = topStatData?.samplePercent
|
||||
|
||||
if (samplePercent && samplePercent < 100) {
|
||||
return (
|
||||
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
|
||||
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
import React, { useState } from "react"
|
||||
import * as api from '../../api'
|
||||
import { getCurrentInterval } from "./interval-picker"
|
||||
|
||||
export default function StatsExport({site, query}) {
|
||||
const [exporting, setExporting] = useState(false)
|
||||
|
||||
function startExport() {
|
||||
setExporting(true)
|
||||
document.cookie = "exporting="
|
||||
pollExportReady()
|
||||
}
|
||||
|
||||
function pollExportReady() {
|
||||
if (document.cookie.includes('exporting')) {
|
||||
setTimeout(pollExportReady, 1000)
|
||||
} else {
|
||||
setExporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
function renderLoading() {
|
||||
return (
|
||||
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function renderExportLink() {
|
||||
const interval = getCurrentInterval(site, query)
|
||||
const queryParams = api.serializeQuery(query, [{ interval }])
|
||||
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
|
||||
|
||||
return (
|
||||
<a href={endpoint} download onClick={startExport}>
|
||||
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
|
||||
<polyline points="7 10 12 15 17 10"></polyline>
|
||||
<line x1="12" y1="15" x2="12" y2="3"></line>
|
||||
</svg>
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-4 h-4 mx-2">
|
||||
{exporting && renderLoading()}
|
||||
{!exporting && renderExportLink()}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
import React from "react"
|
||||
import { parseNaiveDate, isBefore } from '../../util/date'
|
||||
import { Link } from 'react-router-dom'
|
||||
import * as url from '../../util/url'
|
||||
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
|
||||
|
||||
export default function WithImportedSwitch({site, topStatData}) {
|
||||
if (!topStatData?.imports_exist) {
|
||||
return null
|
||||
}
|
||||
|
||||
function isBeforeNativeStats(date) {
|
||||
if (!date) return false
|
||||
|
||||
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
|
||||
const parsedDate = parseNaiveDate(date)
|
||||
|
||||
return isBefore(parsedDate, nativeStatsBegin, "day")
|
||||
}
|
||||
|
||||
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
|
||||
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
|
||||
|
||||
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
|
||||
const withImported = topStatData.with_imported;
|
||||
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
|
||||
const target = url.setQuery('with_imported', !withImported)
|
||||
const tip = withImported ? "" : "do not ";
|
||||
|
||||
return (
|
||||
<Link to={target} className="w-4 h-4 mx-2">
|
||||
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
|
||||
<BarsArrowUpIcon className={"absolute " + toggleColor} />
|
||||
</div>
|
||||
</Link>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
Plausible copyright notice
|
||||
|
||||
This software is Copyright 2024 Plausible Insights OÜ
|
||||
|
||||
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io
|
||||
@ -0,0 +1,42 @@
|
||||
defmodule Plausible.License do
|
||||
@moduledoc """
|
||||
This module ensures that you cannot run Plausible Analytics Enterprise Edition without a valid license key.
|
||||
The software contained within the ee/ and assets/js/dashboard/ee directories are Copyright © Plausible Insights OÜ.
|
||||
We have made this code available solely for informational and transparency purposes. No rights are granted to use,
|
||||
distribute, or exploit this software in any form.
|
||||
|
||||
Any attempt to disable or modify the behavior of this module will be considered a violation of copyright.
|
||||
If you wish to use the Plausible Analytics Enterprise Edition for your own requirements, please contact us
|
||||
at hello@plausible.io to discuss obtaining a license.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
if Mix.env() == :prod do
|
||||
def ensure_valid_license do
|
||||
if has_valid_license?() do
|
||||
:ok
|
||||
else
|
||||
Logger.error(
|
||||
"Invalid or no license key provided for Plausible Enterprise Edition. Please contact hello@plausible.io to acquire a license."
|
||||
)
|
||||
|
||||
Logger.error("Shutting down")
|
||||
System.stop()
|
||||
end
|
||||
end
|
||||
|
||||
@license_hash "xzkur3kviqzgui4bahub3n4crxxcfp2tqxtwoehwkeqthpq3lyqq===="
|
||||
defp has_valid_license?() do
|
||||
hash =
|
||||
:crypto.hash(:sha256, Application.fetch_env!(:plausible, :license_key))
|
||||
|> Base.encode32(case: :lower)
|
||||
|
||||
hash == @license_hash
|
||||
end
|
||||
else
|
||||
def ensure_valid_license do
|
||||
:ok
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240302"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
{
|
||||
"account": "accounts/28425555",
|
||||
"createTime": "2024-02-22T10:50:15.462Z",
|
||||
"currencyCode": "USD",
|
||||
"displayName": "account.one - GA4",
|
||||
"name": "properties/428685444",
|
||||
"parent": "accounts/28425555",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY",
|
||||
"serviceLevel": "GOOGLE_ANALYTICS_STANDARD",
|
||||
"timeZone": "Europe/Warsaw",
|
||||
"updateTime": "2024-02-22T10:50:15.462Z"
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"accountSummaries": [
|
||||
{
|
||||
"account": "accounts/28425178",
|
||||
"displayName": "account.one",
|
||||
"name": "accountSummaries/28425178",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "account.one - GA4",
|
||||
"parent": "accounts/28425178",
|
||||
"property": "properties/428685906",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"account": "accounts/45336102",
|
||||
"displayName": "account.two",
|
||||
"name": "accountSummaries/45336102"
|
||||
},
|
||||
{
|
||||
"account": "accounts/54516992",
|
||||
"displayName": "Demo Account",
|
||||
"name": "accountSummaries/54516992",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "GA4 - Flood-It!",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/153293282",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
},
|
||||
{
|
||||
"displayName": "GA4 - Google Merch Shop",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/213025502",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,786 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Amsterdam"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "totalUsers",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "bounces",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "sessions",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "userEngagementDuration",
|
||||
"type": "TYPE_SECONDS"
|
||||
}
|
||||
],
|
||||
"rowCount": 31,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240131"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "141"
|
||||
},
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "56"
|
||||
},
|
||||
{
|
||||
"value": "146"
|
||||
},
|
||||
{
|
||||
"value": "4815"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240130"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "42"
|
||||
},
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "3481"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240129"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "105"
|
||||
},
|
||||
{
|
||||
"value": "153"
|
||||
},
|
||||
{
|
||||
"value": "54"
|
||||
},
|
||||
{
|
||||
"value": "113"
|
||||
},
|
||||
{
|
||||
"value": "2895"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240128"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "184"
|
||||
},
|
||||
{
|
||||
"value": "259"
|
||||
},
|
||||
{
|
||||
"value": "74"
|
||||
},
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "6861"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240127"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "202"
|
||||
},
|
||||
{
|
||||
"value": "67"
|
||||
},
|
||||
{
|
||||
"value": "158"
|
||||
},
|
||||
{
|
||||
"value": "5259"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240126"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "132"
|
||||
},
|
||||
{
|
||||
"value": "189"
|
||||
},
|
||||
{
|
||||
"value": "57"
|
||||
},
|
||||
{
|
||||
"value": "142"
|
||||
},
|
||||
{
|
||||
"value": "5545"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240125"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "151"
|
||||
},
|
||||
{
|
||||
"value": "229"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "5365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240124"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "137"
|
||||
},
|
||||
{
|
||||
"value": "221"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "148"
|
||||
},
|
||||
{
|
||||
"value": "4454"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240123"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "166"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "4850"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240122"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "161"
|
||||
},
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "73"
|
||||
},
|
||||
{
|
||||
"value": "165"
|
||||
},
|
||||
{
|
||||
"value": "4871"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240121"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "254"
|
||||
},
|
||||
{
|
||||
"value": "397"
|
||||
},
|
||||
{
|
||||
"value": "114"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "8845"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240120"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "214"
|
||||
},
|
||||
{
|
||||
"value": "383"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "6914"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240119"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "381"
|
||||
},
|
||||
{
|
||||
"value": "5571"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "228"
|
||||
},
|
||||
{
|
||||
"value": "98549"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240118"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "177"
|
||||
},
|
||||
{
|
||||
"value": "278"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "186"
|
||||
},
|
||||
{
|
||||
"value": "5553"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240117"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "204"
|
||||
},
|
||||
{
|
||||
"value": "271"
|
||||
},
|
||||
{
|
||||
"value": "97"
|
||||
},
|
||||
{
|
||||
"value": "216"
|
||||
},
|
||||
{
|
||||
"value": "6777"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240116"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "200"
|
||||
},
|
||||
{
|
||||
"value": "280"
|
||||
},
|
||||
{
|
||||
"value": "65"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9381"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240115"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "255"
|
||||
},
|
||||
{
|
||||
"value": "311"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "12365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240114"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "243"
|
||||
},
|
||||
{
|
||||
"value": "302"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "257"
|
||||
},
|
||||
{
|
||||
"value": "11073"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240113"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "231"
|
||||
},
|
||||
{
|
||||
"value": "276"
|
||||
},
|
||||
{
|
||||
"value": "92"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "10084"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240112"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "215"
|
||||
},
|
||||
{
|
||||
"value": "265"
|
||||
},
|
||||
{
|
||||
"value": "75"
|
||||
},
|
||||
{
|
||||
"value": "225"
|
||||
},
|
||||
{
|
||||
"value": "10195"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240111"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "183"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "8200"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240110"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9056"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240109"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "201"
|
||||
},
|
||||
{
|
||||
"value": "290"
|
||||
},
|
||||
{
|
||||
"value": "63"
|
||||
},
|
||||
{
|
||||
"value": "213"
|
||||
},
|
||||
{
|
||||
"value": "11651"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240108"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "209"
|
||||
},
|
||||
{
|
||||
"value": "7946"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240107"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "205"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "80"
|
||||
},
|
||||
{
|
||||
"value": "218"
|
||||
},
|
||||
{
|
||||
"value": "10048"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240106"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "256"
|
||||
},
|
||||
{
|
||||
"value": "303"
|
||||
},
|
||||
{
|
||||
"value": "119"
|
||||
},
|
||||
{
|
||||
"value": "281"
|
||||
},
|
||||
{
|
||||
"value": "8440"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240105"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "226"
|
||||
},
|
||||
{
|
||||
"value": "270"
|
||||
},
|
||||
{
|
||||
"value": "83"
|
||||
},
|
||||
{
|
||||
"value": "237"
|
||||
},
|
||||
{
|
||||
"value": "9311"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240104"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "81"
|
||||
},
|
||||
{
|
||||
"value": "236"
|
||||
},
|
||||
{
|
||||
"value": "12051"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240103"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "283"
|
||||
},
|
||||
{
|
||||
"value": "79"
|
||||
},
|
||||
{
|
||||
"value": "245"
|
||||
},
|
||||
{
|
||||
"value": "10628"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240102"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"value": "210"
|
||||
},
|
||||
{
|
||||
"value": "11168"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240101"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "224"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "197"
|
||||
},
|
||||
{
|
||||
"value": "8080"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240222"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20200421"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20120118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20170118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,136 @@
|
||||
defmodule Plausible.Cache.Adapter do
|
||||
@moduledoc """
|
||||
Interface for the underlying cache implementation.
|
||||
Currently: ConCache
|
||||
|
||||
Using the Adapter module directly, the user must ensure that the relevant
|
||||
processes are available to use, which is normally done via the child specification.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()
|
||||
def child_spec(name, child_id, opts \\ [])
|
||||
when is_atom(name) and is_atom(child_id) and is_list(opts) do
|
||||
cache_name = Keyword.get(opts, :cache_name, name)
|
||||
child_id = Keyword.get(opts, :child_id, child_id)
|
||||
ttl_check_interval = Keyword.get(opts, :ttl_check_interval, false)
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Keyword.put(:name, cache_name)
|
||||
|> Keyword.put(:ttl_check_interval, ttl_check_interval)
|
||||
|
||||
Supervisor.child_spec(
|
||||
{ConCache, opts},
|
||||
id: child_id
|
||||
)
|
||||
end
|
||||
|
||||
@spec size(atom()) :: non_neg_integer() | nil
|
||||
def size(cache_name) do
|
||||
ConCache.size(cache_name)
|
||||
catch
|
||||
:exit, _ -> nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any()) :: any()
|
||||
def get(cache_name, key) do
|
||||
result = ConCache.get(cache_name, key)
|
||||
if is_nil(result), do: miss(cache_name), else: hit(cache_name)
|
||||
result
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any(), (-> any())) :: any()
|
||||
def get(cache_name, key, fallback_fn) do
|
||||
cache = ConCache.Owner.cache(cache_name)
|
||||
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
get_or_store_isolated(cache, cache_name, key, fallback_fn)
|
||||
|
||||
value ->
|
||||
hit(cache_name)
|
||||
value
|
||||
end
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put(atom(), any(), any()) :: any()
|
||||
def put(cache_name, key, value) do
|
||||
:ok = ConCache.put(cache_name, key, value)
|
||||
value
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting a key to '#{cache_name}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put_many(atom(), [any()]) :: :ok
|
||||
def put_many(cache_name, items) when is_list(items) do
|
||||
true = :ets.insert(ConCache.ets(cache_name), items)
|
||||
:ok
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting keys to '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec delete(atom(), any()) :: :ok
|
||||
def delete(cache_name, key) do
|
||||
ConCache.dirty_delete(cache_name, key)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error deleting a key in '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec keys(atom()) :: Enumerable.t()
|
||||
def keys(cache_name) do
|
||||
ets = ConCache.ets(cache_name)
|
||||
|
||||
Stream.resource(
|
||||
fn -> :ets.first(ets) end,
|
||||
fn
|
||||
:"$end_of_table" -> {:halt, nil}
|
||||
prev_key -> {[prev_key], :ets.next(ets, prev_key)}
|
||||
end,
|
||||
fn _ -> :ok end
|
||||
)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
[]
|
||||
end
|
||||
|
||||
defp hit(cache_name) do
|
||||
Plausible.Cache.Stats.record_hit(cache_name)
|
||||
end
|
||||
|
||||
defp miss(cache_name) do
|
||||
Plausible.Cache.Stats.record_miss(cache_name)
|
||||
end
|
||||
|
||||
defp get_or_store_isolated(cache, cache_name, key, fun) do
|
||||
ConCache.isolated(cache_name, key, fn ->
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
new_value = fun.()
|
||||
ConCache.Operations.dirty_put(cache, key, new_value)
|
||||
miss(cache_name)
|
||||
new_value
|
||||
|
||||
existing ->
|
||||
hit(cache_name)
|
||||
existing
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,84 @@
|
||||
defmodule Plausible.Cache.Stats do
|
||||
@moduledoc """
|
||||
Keeps track of hit/miss ratio for various caches.
|
||||
"""
|
||||
|
||||
use GenServer
|
||||
|
||||
@hit :hit
|
||||
@miss :miss
|
||||
@telemetry_hit [:plausible, :cache, :adapter, @hit]
|
||||
@telemetry_miss [:plausible, :cache, :adapter, @miss]
|
||||
@telemetry_events [@telemetry_hit, @telemetry_miss]
|
||||
|
||||
def start_link(_opts) do
|
||||
GenServer.start_link(__MODULE__, nil)
|
||||
end
|
||||
|
||||
def record_hit(cache_name) do
|
||||
:telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def record_miss(cache_name) do
|
||||
:telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def init(nil) do
|
||||
__MODULE__ =
|
||||
:ets.new(__MODULE__, [
|
||||
:public,
|
||||
:named_table,
|
||||
:set,
|
||||
read_concurrency: true,
|
||||
write_concurrency: true
|
||||
])
|
||||
|
||||
:telemetry.attach_many(
|
||||
"plausible-cache-stats",
|
||||
@telemetry_events,
|
||||
&__MODULE__.handle_telemetry_event/4,
|
||||
nil
|
||||
)
|
||||
|
||||
{:ok, nil}
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @hit)
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @miss)
|
||||
end
|
||||
|
||||
def gather(cache_name) do
|
||||
{:ok,
|
||||
%{
|
||||
hit_rate: hit_rate(cache_name),
|
||||
count: size(cache_name) || 0
|
||||
}}
|
||||
end
|
||||
|
||||
defdelegate size(cache_name), to: Plausible.Cache.Adapter
|
||||
|
||||
def bump(cache_name, type) do
|
||||
:ets.update_counter(
|
||||
__MODULE__,
|
||||
{cache_name, type},
|
||||
1,
|
||||
{{cache_name, type}, 0}
|
||||
)
|
||||
end
|
||||
|
||||
def hit_rate(cache_name) do
|
||||
hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)
|
||||
miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)
|
||||
hit_miss = hit + miss
|
||||
|
||||
if hit_miss == 0 do
|
||||
0.0
|
||||
else
|
||||
hit / hit_miss * 100
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,166 @@
|
||||
defmodule Plausible.DataMigration.PopulateEventSessionColumns do
|
||||
@moduledoc """
|
||||
Populates event session columns with data from sessions table.
|
||||
|
||||
Run via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.run"
|
||||
Kill via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.kill"
|
||||
Monitor via ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.report_progress"
|
||||
|
||||
Suggested to run in a screen/tmux session to be able to easily monitor
|
||||
|
||||
SQL files available at: priv/data_migrations/PopulateEventSessionColumns/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "PopulateEventSessionColumns", repo: Plausible.IngestRepo
|
||||
|
||||
require Logger
|
||||
|
||||
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#cache for meaning of these defaults
|
||||
@default_dictionary_config %{
|
||||
lifetime: 600_000,
|
||||
size_in_cells: 1_000_000,
|
||||
max_threads_for_updates: 6
|
||||
}
|
||||
|
||||
def run(opts \\ []) do
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("create-sessions-dictionary",
|
||||
cluster?: cluster?,
|
||||
dictionary_connection_params:
|
||||
Keyword.get(opts, :dictionary_connection_string, dictionary_connection_params()),
|
||||
dictionary_config: dictionary_config(opts)
|
||||
)
|
||||
|
||||
{partitions, _, _, _} = get_partitions(opts)
|
||||
|
||||
IO.puts("Starting mutation on #{length(partitions)} partition(s)")
|
||||
|
||||
for partition <- partitions do
|
||||
{:ok, _} =
|
||||
run_sql("update-table", [cluster?: cluster?, partition: partition],
|
||||
settings: [allow_nondeterministic_mutations: 1]
|
||||
)
|
||||
end
|
||||
|
||||
wait_until_mutations_complete(opts)
|
||||
|
||||
IO.puts("Mutations seem done, cleaning up!")
|
||||
{:ok, _} = run_sql("drop-sessions-dictionary", cluster?: cluster?)
|
||||
end
|
||||
|
||||
def kill(opts \\ []) do
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")
|
||||
|
||||
report_progress(opts)
|
||||
|
||||
IO.puts("Killing running mutations")
|
||||
{:ok, _} = run_sql("kill-running-mutations", cluster?: cluster?)
|
||||
end
|
||||
|
||||
def wait_until_mutations_complete(opts \\ []) do
|
||||
Process.sleep(5_000)
|
||||
in_progress? = report_progress(opts)
|
||||
|
||||
if in_progress? do
|
||||
wait_until_mutations_complete(opts)
|
||||
end
|
||||
end
|
||||
|
||||
def report_progress(opts \\ []) do
|
||||
{partitions, parts, min_partition, max_partition} = get_partitions(opts)
|
||||
|
||||
{:ok, %{rows: mutation_results}} =
|
||||
run_sql("get-mutations-progress",
|
||||
min_partition: min_partition,
|
||||
max_partition: max_partition
|
||||
)
|
||||
|
||||
[
|
||||
[
|
||||
mutations,
|
||||
parts_to_do,
|
||||
running_for,
|
||||
total_size,
|
||||
todo_size,
|
||||
progress,
|
||||
latest_fail_reason,
|
||||
_,
|
||||
_
|
||||
]
|
||||
] =
|
||||
mutation_results
|
||||
|
||||
{:ok, %{rows: [[merges]]}} = run_sql("get-merges-progress")
|
||||
{:ok, %{rows: disks}} = run_sql("get-disks")
|
||||
|
||||
IO.puts("\n\n#{Timex.now() |> Timex.format!("{ISO:Extended}")}")
|
||||
|
||||
# List partitions that need to run
|
||||
IO.puts(
|
||||
"Progress report for partitions #{Enum.min(partitions)}-#{Enum.max(partitions)} (parts: #{length(parts)})"
|
||||
)
|
||||
|
||||
IO.puts("Disks overview:")
|
||||
|
||||
for [name, path, full_space, total_space, full_percentage] <- disks do
|
||||
IO.puts(
|
||||
" #{name} at #{path} is at #{full_space}/#{total_space} (#{full_percentage}% full)"
|
||||
)
|
||||
end
|
||||
|
||||
IO.puts("Currently #{mutations} mutation(s) are running.")
|
||||
|
||||
if mutations > 0 do
|
||||
IO.puts(" To do #{parts_to_do} parts, #{todo_size}")
|
||||
IO.puts(" Out of #{length(parts)} parts, #{total_size}")
|
||||
IO.puts(" Running for #{format_duration(running_for)}")
|
||||
|
||||
if progress > 0 do
|
||||
estimated_time_left = round(running_for / progress / 100 - running_for)
|
||||
IO.puts(" Estimated #{progress}% done, #{format_duration(estimated_time_left)} left")
|
||||
end
|
||||
|
||||
if latest_fail_reason do
|
||||
IO.puts(" Some mutations might be failing. ClickHouse report: #{latest_fail_reason}")
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Currently #{merges} merge(s) are running relating to mutations.")
|
||||
|
||||
mutations > 0
|
||||
end
|
||||
|
||||
defp dictionary_config(opts) do
|
||||
@default_dictionary_config
|
||||
|> Map.merge(Keyword.get(opts, :dictionary_config, %{}))
|
||||
end
|
||||
|
||||
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context
|
||||
defp dictionary_connection_params() do
|
||||
Plausible.IngestRepo.config()
|
||||
|> Enum.map(fn
|
||||
{:database, database} -> "DB '#{database}'"
|
||||
{:username, username} -> "USER '#{username}'"
|
||||
{:password, password} -> "PASSWORD '#{password}'"
|
||||
_ -> nil
|
||||
end)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.join(" ")
|
||||
end
|
||||
|
||||
defp get_partitions(opts) do
|
||||
[min_partition, max_partition] = Keyword.get(opts, :partition_range, ["0", "999999"])
|
||||
|
||||
{:ok, %{rows: [[partitions, parts]]}} =
|
||||
run_sql("list-partitions", min_partition: min_partition, max_partition: max_partition)
|
||||
|
||||
{partitions, parts, min_partition, max_partition}
|
||||
end
|
||||
|
||||
defp format_duration(seconds) do
|
||||
seconds
|
||||
|> Timex.Duration.from_seconds()
|
||||
|> Timex.format_duration(Timex.Format.Duration.Formatters.Humanized)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,188 @@
|
||||
defmodule Plausible.DataMigration.SiteImports do
|
||||
@moduledoc """
|
||||
Site imports migration backfilling SiteImport entries for old imports
|
||||
and alters import end dates to match actual end date of respective import stats.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.ClickhouseRepo
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
def run(opts \\ []) do
|
||||
dry_run? = Keyword.get(opts, :dry_run?, true)
|
||||
|
||||
site_import_query =
|
||||
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
|
||||
|
||||
sites_with_imports =
|
||||
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|
||||
|> Repo.all(log: false)
|
||||
|
||||
sites_count = length(sites_with_imports)
|
||||
|
||||
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
|
||||
|
||||
for {site, idx} <- Enum.with_index(sites_with_imports) do
|
||||
site_imports =
|
||||
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all(log: false)
|
||||
|
||||
IO.puts(
|
||||
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
|
||||
)
|
||||
|
||||
site_imports =
|
||||
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
|
||||
IO.puts("Creating legacy site import entry for site ID #{site.id}")
|
||||
|
||||
# create legacy entry if there's not one yet
|
||||
params =
|
||||
site.imported_data
|
||||
|> Imported.SiteImport.from_legacy()
|
||||
|> Map.put(:site_id, site.id)
|
||||
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
|
||||
|
||||
legacy_site_import =
|
||||
%Imported.SiteImport{}
|
||||
|> Ecto.Changeset.change(params)
|
||||
|> insert!(dry_run?)
|
||||
|
||||
[legacy_site_import | site_imports]
|
||||
else
|
||||
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
|
||||
|
||||
site_imports
|
||||
end
|
||||
|
||||
# adjust end date for each site import
|
||||
for site_import <- site_imports do
|
||||
IO.puts(
|
||||
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
|
||||
)
|
||||
|
||||
import_ids =
|
||||
if site_import.legacy do
|
||||
[0, site_import.id]
|
||||
else
|
||||
[site_import.id]
|
||||
end
|
||||
|
||||
end_date = imported_stats_end_date(site.id, import_ids)
|
||||
|
||||
if !end_date do
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
|
||||
)
|
||||
|
||||
if site_import.legacy do
|
||||
clear_imported_data(site, dry_run?)
|
||||
end
|
||||
|
||||
delete!(site_import, dry_run?)
|
||||
else
|
||||
end_date =
|
||||
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
|
||||
end_date
|
||||
else
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
|
||||
)
|
||||
|
||||
site_import.end_date
|
||||
end
|
||||
|
||||
site_import
|
||||
|> Ecto.Changeset.change(end_date: end_date)
|
||||
|> update!(dry_run?)
|
||||
|
||||
IO.puts(
|
||||
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Done processing site ID #{site.id}")
|
||||
end
|
||||
|
||||
IO.puts("Finished")
|
||||
end
|
||||
|
||||
# Exposed for testing purposes
|
||||
@doc false
|
||||
def imported_stats_end_date(site_id, import_ids) do
|
||||
[first_schema | schemas] = Imported.schemas()
|
||||
|
||||
query =
|
||||
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
|
||||
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
|
||||
end)
|
||||
|
||||
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
|
||||
|
||||
if dates != [] do
|
||||
case Enum.max(dates, Date) do
|
||||
# no stats for this domain yet
|
||||
~D[1970-01-01] ->
|
||||
nil
|
||||
|
||||
date ->
|
||||
date
|
||||
end
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp insert!(changeset, false = _dry_run?) do
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
defp insert!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
changeset
|
||||
|> Ecto.Changeset.change(id: 0)
|
||||
|> Ecto.Changeset.apply_changes()
|
||||
else
|
||||
raise "Invalid insert: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, false = _dry_run?) do
|
||||
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, true = _dry_run?) do
|
||||
%{site | imported_data: nil}
|
||||
end
|
||||
|
||||
defp update!(changeset, false = _dry_run?) do
|
||||
Repo.update!(changeset)
|
||||
end
|
||||
|
||||
defp update!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
Ecto.Changeset.apply_changes(changeset)
|
||||
else
|
||||
raise "Invalid update: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp delete!(entity, false = _dry_run?) do
|
||||
Repo.delete!(entity)
|
||||
end
|
||||
|
||||
defp delete!(entity, true = _dry_run?) do
|
||||
entity
|
||||
end
|
||||
|
||||
defp max_date_query(schema, site_id, import_ids) do
|
||||
from(q in schema,
|
||||
where: q.site_id == ^site_id,
|
||||
where: q.import_id in ^import_ids,
|
||||
select: %{max_date: fragment("max(?)", q.date)}
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,75 @@
|
||||
defmodule Plausible.DataMigration.VersionedSessions do
|
||||
@moduledoc """
|
||||
Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
|
||||
priv/data_migrations/VersionedSessions/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
|
||||
|
||||
@suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
|
||||
@versioned_table_engines [
|
||||
"ReplicatedVersionedCollapsingMergeTree",
|
||||
"VersionedCollapsingMergeTree"
|
||||
]
|
||||
|
||||
def run(opts \\ []) do
|
||||
run_exchange? = Keyword.get(opts, :run_exchange?, true)
|
||||
|
||||
unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
|
||||
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
|
||||
|
||||
{:ok, %{rows: partitions}} = run_sql("list-partitions")
|
||||
partitions = Enum.map(partitions, fn [part] -> part end)
|
||||
|
||||
{:ok, %{rows: [[current_table_engine, table_settings]]}} =
|
||||
run_sql("get-sessions-table-settings")
|
||||
|
||||
if Enum.member?(@versioned_table_engines, current_table_engine) do
|
||||
IO.puts("sessions_v2 table is already versioned, no migration needed")
|
||||
else
|
||||
{:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("create-sessions-tmp-table",
|
||||
cluster?: cluster?,
|
||||
table_settings: table_settings,
|
||||
unique_suffix: unique_suffix
|
||||
)
|
||||
|
||||
for partition <- partitions do
|
||||
{:ok, _} = run_sql("attach-partition", partition: partition)
|
||||
end
|
||||
|
||||
if run_exchange? do
|
||||
run_exchange(cluster?)
|
||||
end
|
||||
|
||||
IO.puts("Migration done!")
|
||||
end
|
||||
end
|
||||
|
||||
defp run_exchange(cluster?) do
|
||||
case run_sql("exchange-sessions-tables", cluster?: cluster?) do
|
||||
{:ok, _} ->
|
||||
nil
|
||||
|
||||
# Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
|
||||
{:error, %Ch.Error{code: 1}} ->
|
||||
IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2",
|
||||
to: "sessions_v2_backup",
|
||||
cluster?: cluster?
|
||||
)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2_tmp_versioned",
|
||||
to: "sessions_v2",
|
||||
cluster?: cluster?
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,371 @@
|
||||
defmodule Plausible.Exports do
|
||||
@moduledoc """
|
||||
Contains functions to export data for events and sessions as Zip archives.
|
||||
"""
|
||||
|
||||
require Plausible
|
||||
import Ecto.Query
|
||||
|
||||
@doc """
|
||||
Renders filename for the Zip archive containing the exported CSV files.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"plausible_io_20210101_20241231.zip"
|
||||
|
||||
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"Bücher_example_20210101_20241231.zip"
|
||||
|
||||
"""
|
||||
def archive_filename(domain, min_date, max_date) do
|
||||
name =
|
||||
Enum.join(
|
||||
[
|
||||
String.replace(domain, ".", "_"),
|
||||
Calendar.strftime(min_date, "%Y%m%d"),
|
||||
Calendar.strftime(max_date, "%Y%m%d")
|
||||
],
|
||||
"_"
|
||||
)
|
||||
|
||||
name <> ".zip"
|
||||
end
|
||||
|
||||
@doc """
|
||||
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
|
||||
tables into the format of `imported_*` tables for a website.
|
||||
"""
|
||||
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
|
||||
%{String.t() => Ecto.Query.t()}
|
||||
def export_queries(site_id, opts \\ []) do
|
||||
extname = opts[:extname] || ".csv"
|
||||
date_range = opts[:date_range]
|
||||
|
||||
filename = fn table ->
|
||||
name =
|
||||
if date_range do
|
||||
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
|
||||
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
|
||||
"#{table}_#{first_date}_#{last_date}"
|
||||
else
|
||||
table
|
||||
end
|
||||
|
||||
name <> extname
|
||||
end
|
||||
|
||||
%{
|
||||
filename.("imported_visitors") => export_visitors_q(site_id),
|
||||
filename.("imported_sources") => export_sources_q(site_id),
|
||||
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
|
||||
filename.("imported_pages") => export_pages_q(site_id),
|
||||
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
|
||||
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
|
||||
filename.("imported_locations") => export_locations_q(site_id),
|
||||
filename.("imported_devices") => export_devices_q(site_id),
|
||||
filename.("imported_browsers") => export_browsers_q(site_id),
|
||||
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
|
||||
}
|
||||
end
|
||||
|
||||
Plausible.on_full_build do
|
||||
defp sampled(table) do
|
||||
Plausible.Stats.Sampling.add_query_hint(from(table))
|
||||
end
|
||||
else
|
||||
defp sampled(table) do
|
||||
table
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop date(timestamp) do
|
||||
quote do
|
||||
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visit_duration(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
|
||||
:visit_duration
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visitors(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
|
||||
:visitors
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visits(t) do
|
||||
quote do
|
||||
selected_as(sum(unquote(t).sign), :visits)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop bounces(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
|
||||
:bounces
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop pageviews(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
|
||||
:pageviews
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_visitors_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
visitors(s),
|
||||
pageviews(s),
|
||||
bounces(s),
|
||||
visits(s),
|
||||
visit_duration(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_sources_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [
|
||||
selected_as(:date),
|
||||
selected_as(:source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term
|
||||
],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.referrer_source, :source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term,
|
||||
pageviews(s),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_pages_q(site_id) do
|
||||
window_q =
|
||||
from e in sampled("events_v2"),
|
||||
where: e.site_id == ^site_id,
|
||||
select: %{
|
||||
timestamp: e.timestamp,
|
||||
next_timestamp:
|
||||
over(fragment("leadInFrame(?)", e.timestamp),
|
||||
partition_by: e.session_id,
|
||||
order_by: e.timestamp,
|
||||
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
|
||||
),
|
||||
pathname: e.pathname,
|
||||
hostname: e.hostname,
|
||||
name: e.name,
|
||||
user_id: e.user_id,
|
||||
session_id: e.session_id,
|
||||
_sample_factor: fragment("_sample_factor")
|
||||
}
|
||||
|
||||
from e in subquery(window_q),
|
||||
group_by: [selected_as(:date), e.pathname],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(e.timestamp),
|
||||
selected_as(fragment("any(?)", e.hostname), :hostname),
|
||||
selected_as(e.pathname, :page),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
|
||||
:visits
|
||||
),
|
||||
visitors(e),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
),
|
||||
# NOTE: are exits pageviews or any events?
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
|
||||
:exits
|
||||
),
|
||||
selected_as(
|
||||
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
|
||||
:time_on_page
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_entry_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.entry_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.entry_page,
|
||||
visitors(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:entrances
|
||||
),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_exit_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.exit_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.exit_page,
|
||||
visitors(s),
|
||||
visit_duration(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:exits
|
||||
),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_locations_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
|
||||
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.country_code, :country),
|
||||
selected_as(s.subdivision1_code, :region),
|
||||
selected_as(s.city_geoname_id, :city),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_devices_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.screen_size],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.screen_size, :device),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_browsers_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.browser, s.browser_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.browser,
|
||||
s.browser_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_operating_systems_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.operating_system,
|
||||
s.operating_system_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Creates a streamable Zip archive from the provided (named) Ecto queries.
|
||||
|
||||
Example usage:
|
||||
|
||||
{:ok, pool} = Ch.start_link(pool_size: 1)
|
||||
|
||||
DBConnection.run(pool, fn conn ->
|
||||
conn
|
||||
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!("export.zip"))
|
||||
|> Stream.run()
|
||||
end)
|
||||
|
||||
"""
|
||||
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
|
||||
Enumerable.t()
|
||||
def stream_archive(conn, named_queries, opts \\ []) do
|
||||
entries =
|
||||
Enum.map(named_queries, fn {name, query} ->
|
||||
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
|
||||
|
||||
datastream =
|
||||
conn
|
||||
|> Ch.stream(sql, params, opts)
|
||||
|> Stream.map(fn %Ch.Result{data: data} -> data end)
|
||||
|
||||
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
|
||||
end)
|
||||
|
||||
Zstream.zip(entries)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,152 @@
|
||||
defmodule Plausible.Google.GA4.API do
|
||||
@moduledoc """
|
||||
API for Google Analytics 4.
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.GA4
|
||||
|
||||
require Logger
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 50_000
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
def list_properties(access_token) do
|
||||
case GA4.HTTP.list_accounts_for_user(access_token) do
|
||||
{:ok, %{"accountSummaries" => accounts}} ->
|
||||
accounts =
|
||||
accounts
|
||||
|> Enum.filter(& &1["propertySummaries"])
|
||||
|> Enum.map(fn account ->
|
||||
%{"account" => account_id, "displayName" => account_name} = account
|
||||
|
||||
{"#{account_name} (#{account_id})",
|
||||
Enum.map(account["propertySummaries"], fn property ->
|
||||
%{"displayName" => property_name, "property" => property_id} = property
|
||||
|
||||
{"#{property_name} (#{property_id})", property_id}
|
||||
end)}
|
||||
end)
|
||||
|
||||
{:ok, accounts}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, lookup_property) do
|
||||
case GA4.HTTP.get_property(access_token, lookup_property) do
|
||||
{:ok, property} ->
|
||||
%{"displayName" => property_name, "name" => property_id, "account" => account_id} =
|
||||
property
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
id: property_id,
|
||||
name: "#{property_name} (#{property_id})",
|
||||
account_id: account_id
|
||||
}}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_start_date(access_token, property)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_end_date(access_token, property)
|
||||
end
|
||||
|
||||
def import_analytics(date_range, property, auth, persist_fn) do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
|
||||
)
|
||||
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, property, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, property, access_token, persist_fn) do
|
||||
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
|
||||
)
|
||||
|
||||
report_request = prepare_request(report_request, date_range, property, access_token)
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case GA4.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, row_count}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
if report_request.offset + @per_page < row_count do
|
||||
fetch_and_persist(
|
||||
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
|
||||
)
|
||||
|
||||
{:error, cause}
|
||||
else
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
|
||||
)
|
||||
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp prepare_request(report_request, date_range, property, access_token) do
|
||||
%GA4.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
property: property,
|
||||
access_token: access_token,
|
||||
offset: 0,
|
||||
limit: @per_page
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,244 @@
|
||||
defmodule Plausible.Google.GA4.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Google Analytics 4 API.
|
||||
"""
|
||||
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
require Logger
|
||||
|
||||
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
|
||||
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: report_request.property,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
|
||||
metrics: Enum.map(report_request.metrics, &build_metric/1),
|
||||
orderBys: [
|
||||
%{
|
||||
dimension: %{
|
||||
dimensionName: "date",
|
||||
orderType: "ALPHANUMERIC"
|
||||
},
|
||||
desc: true
|
||||
}
|
||||
],
|
||||
limit: report_request.limit,
|
||||
offset: report_request.offset
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url =
|
||||
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
url,
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
row_count <- Map.fetch!(report, "rowCount"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, row_count}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
|
||||
)
|
||||
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp build_metric(expression) do
|
||||
case String.split(expression, " = ") do
|
||||
[name, expression] ->
|
||||
%{
|
||||
name: name,
|
||||
expression: expression
|
||||
}
|
||||
|
||||
[name] ->
|
||||
%{name: name}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"rows" => rows,
|
||||
"dimensionHeaders" => dimension_headers,
|
||||
"metricHeaders" => metric_headers
|
||||
})
|
||||
when is_list(rows) do
|
||||
dimension_headers = Enum.map(dimension_headers, & &1["name"])
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
|
||||
dimension_values = Enum.map(dimensions, & &1["value"])
|
||||
metric_values = Enum.map(metrics, & &1["value"])
|
||||
metrics = Enum.zip(metric_headers, metric_values)
|
||||
dimensions = Enum.zip(dimension_headers, dimension_values)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_accounts_for_user(access_token) do
|
||||
url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, property) do
|
||||
url = "#{admin_api_url()}/v1beta/#{property}"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
|
||||
{:error, :not_found}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 property #{property}",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2015-08-14"
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, property, edge) do
|
||||
descending? = edge == :end
|
||||
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: "#{property}",
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "date"}],
|
||||
metrics: [%{name: "screenPageViews"}],
|
||||
orderBys: [
|
||||
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
|
||||
],
|
||||
limit: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["rows"] do
|
||||
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
|
||||
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
|
||||
end
|
||||
@ -0,0 +1,135 @@
|
||||
defmodule Plausible.Google.GA4.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Google Analytics 4 API
|
||||
"""
|
||||
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
:metrics,
|
||||
:date_range,
|
||||
:property,
|
||||
:access_token,
|
||||
:offset,
|
||||
:limit
|
||||
]
|
||||
|
||||
@type t() :: %__MODULE__{
|
||||
dataset: String.t(),
|
||||
dimensions: [String.t()],
|
||||
metrics: [String.t()],
|
||||
date_range: Date.Range.t(),
|
||||
property: term(),
|
||||
access_token: String.t(),
|
||||
offset: non_neg_integer(),
|
||||
limit: non_neg_integer()
|
||||
}
|
||||
|
||||
def full_report do
|
||||
[
|
||||
%__MODULE__{
|
||||
dataset: "imported_visitors",
|
||||
dimensions: ["date"],
|
||||
metrics: [
|
||||
"totalUsers",
|
||||
"screenPageViews",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"sessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_sources",
|
||||
dimensions: [
|
||||
"date",
|
||||
"sessionSource",
|
||||
"sessionMedium",
|
||||
"sessionCampaignName",
|
||||
"sessionManualAdContent",
|
||||
"sessionGoogleAdsKeyword",
|
||||
"pageReferrer"
|
||||
],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_pages",
|
||||
dimensions: ["date", "hostName", "pagePath"],
|
||||
# NOTE: no exits as GA4 DATA API does not provide that metric
|
||||
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_entry_pages",
|
||||
dimensions: ["date", "landingPage"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"userEngagementDuration",
|
||||
"bounces = sessions - engagedSessions"
|
||||
]
|
||||
},
|
||||
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
|
||||
# %__MODULE__{
|
||||
# dataset: "imported_exit_pages",
|
||||
# dimensions: ["date", "ga:exitPagePath"],
|
||||
# metrics: [
|
||||
# "totalUsers",
|
||||
# "sessions",
|
||||
# "screenPageViews",
|
||||
# "userEngagementDuration",
|
||||
# "bounces = sessions - engagedSessions"
|
||||
# ]
|
||||
# },
|
||||
%__MODULE__{
|
||||
dataset: "imported_locations",
|
||||
dimensions: ["date", "countryId", "region", "city"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_devices",
|
||||
dimensions: ["date", "deviceCategory"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_browsers",
|
||||
dimensions: ["date", "browser"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_operating_systems",
|
||||
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
}
|
||||
]
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,155 @@
|
||||
defmodule Plausible.Google.UA.API do
|
||||
@moduledoc """
|
||||
API for Universal Analytics
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.UA
|
||||
|
||||
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 7_500
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
@spec list_views(access_token :: String.t()) ::
|
||||
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
|
||||
@doc """
|
||||
Lists Google Analytics views grouped by hostname.
|
||||
"""
|
||||
def list_views(access_token) do
|
||||
case UA.HTTP.list_views_for_user(access_token) do
|
||||
{:ok, %{"items" => views}} ->
|
||||
views =
|
||||
views
|
||||
|> Enum.group_by(&view_hostname/1, &view_names/1)
|
||||
|> Enum.sort_by(fn {key, _} -> key end)
|
||||
|
||||
{:ok, views}
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
|
||||
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
|
||||
@doc """
|
||||
Returns a single Google Analytics view if the user has access to it.
|
||||
"""
|
||||
def get_view(access_token, lookup_id) do
|
||||
with {:ok, views} <- list_views(access_token) do
|
||||
views =
|
||||
views
|
||||
|> Enum.map(&elem(&1, 1))
|
||||
|> List.flatten()
|
||||
|
||||
case Enum.find(views, fn {_name, id} -> id == lookup_id end) do
|
||||
{view_name, view_id} ->
|
||||
{:ok, %{id: view_id, name: "#{view_name}"}}
|
||||
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_start_date(access_token, view_id)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_end_date(access_token, view_id)
|
||||
end
|
||||
|
||||
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
|
||||
:ok | {:error, term()}
|
||||
@doc """
|
||||
Imports stats from a Google Analytics UA view to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics reports in batches of #{@per_page} per
|
||||
request. The batches are then passed to persist callback.
|
||||
|
||||
Requests to Google Analytics can fail, and are retried at most
|
||||
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
|
||||
importing has finished or `{:error, term()}` when a request to GA failed too
|
||||
many times.
|
||||
|
||||
Useful links:
|
||||
|
||||
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
|
||||
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
|
||||
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
|
||||
|
||||
"""
|
||||
def import_analytics(date_range, view_id, auth, persist_fn) do
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, view_id, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
|
||||
Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
report_request = %UA.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
view_id: view_id,
|
||||
access_token: access_token,
|
||||
page_token: nil,
|
||||
page_size: @per_page
|
||||
}
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case UA.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, next_page_token}} ->
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
if next_page_token do
|
||||
fetch_and_persist(
|
||||
%UA.ReportRequest{report_request | page_token: next_page_token},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
{:error, cause}
|
||||
else
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp view_hostname(view) do
|
||||
case view do
|
||||
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
|
||||
_any -> "Others"
|
||||
end
|
||||
end
|
||||
|
||||
defp view_names(%{"name" => name, "id" => id}) do
|
||||
{"#{id} - #{name}", id}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,191 @@
|
||||
defmodule Plausible.Google.UA.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Universal Analytics API.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
@spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
|
||||
{:ok, {[map()], String.t() | nil}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: report_request.view_id,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
|
||||
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
|
||||
pageSize: report_request.page_size,
|
||||
pageToken: report_request.page_token
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
"#{reporting_api_url()}/v4/reports:batchGet",
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
token <- Map.get(report, "nextPageToken"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, token}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, _reason} ->
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"data" => %{} = data,
|
||||
"columnHeader" => %{
|
||||
"dimensions" => dimension_headers,
|
||||
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
|
||||
}
|
||||
}) do
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
rows = Map.get(data, "rows", [])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
|
||||
metrics = Enum.zip(metric_headers, metrics)
|
||||
dimensions = Enum.zip(dimension_headers, dimensions)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_views_for_user(access_token) do
|
||||
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2005-01-01"
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, view_id, edge) do
|
||||
sort_order =
|
||||
if edge == :start do
|
||||
"ASCENDING"
|
||||
else
|
||||
"DESCENDING"
|
||||
end
|
||||
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: view_id,
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "ga:date", histogramBuckets: []}],
|
||||
metrics: [%{expression: "ga:pageviews"}],
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}],
|
||||
pageSize: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v4/reports:batchGet"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["data"]["rows"] do
|
||||
[%{"dimensions" => [date_str]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving UA #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp config, do: Application.get_env(:plausible, :google)
|
||||
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
|
||||
defp api_url, do: Keyword.fetch!(config(), :api_url)
|
||||
end
|
||||
@ -1,4 +1,8 @@
|
||||
defmodule Plausible.Google.ReportRequest do
|
||||
defmodule Plausible.Google.UA.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Universal Analytics API
|
||||
"""
|
||||
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
@ -0,0 +1,222 @@
|
||||
defmodule Plausible.Imported do
|
||||
@moduledoc """
|
||||
Context for managing site statistics imports.
|
||||
|
||||
Currently following importers are implemented:
|
||||
|
||||
* `Plausible.Imported.UniversalAnalytics` - existing mechanism, for legacy Google
|
||||
analytics formerly known as "Google Analytics"
|
||||
* `Plausible.Imported.NoopImporter` - importer stub, used mainly for testing purposes
|
||||
* `Plausible.Imported.CSVImporter` - CSV importer from S3
|
||||
|
||||
For more information on implementing importers, see `Plausible.Imported.Importer`.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Imported.SiteImport
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
require Plausible.Imported.SiteImport
|
||||
|
||||
@tables [
|
||||
Imported.Visitor,
|
||||
Imported.Source,
|
||||
Imported.Page,
|
||||
Imported.EntryPage,
|
||||
Imported.ExitPage,
|
||||
Imported.Location,
|
||||
Imported.Device,
|
||||
Imported.Browser,
|
||||
Imported.OperatingSystem
|
||||
]
|
||||
|
||||
@table_names Enum.map(@tables, & &1.__schema__(:source))
|
||||
# Maximum number of complete imports to account for when querying stats
|
||||
@max_complete_imports 5
|
||||
|
||||
@spec schemas() :: [module()]
|
||||
def schemas, do: @tables
|
||||
|
||||
@spec tables() :: [String.t()]
|
||||
def tables, do: @table_names
|
||||
|
||||
@spec max_complete_imports() :: non_neg_integer()
|
||||
def max_complete_imports(), do: @max_complete_imports
|
||||
|
||||
@spec load_import_data(Site.t()) :: Site.t()
|
||||
def load_import_data(%{import_data_loaded: true} = site), do: site
|
||||
|
||||
def load_import_data(site) do
|
||||
complete_import_ids = list_complete_import_ids(site)
|
||||
dates = get_imports_date_range(site)
|
||||
|
||||
%{
|
||||
site
|
||||
| import_data_loaded: true,
|
||||
earliest_import_start_date: dates.start_date,
|
||||
latest_import_end_date: dates.end_date,
|
||||
complete_import_ids: complete_import_ids
|
||||
}
|
||||
end
|
||||
|
||||
@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
|
||||
def get_import(site, import_id) do
|
||||
Repo.get_by(SiteImport, id: import_id, site_id: site.id)
|
||||
end
|
||||
|
||||
defdelegate listen(), to: Imported.Importer
|
||||
|
||||
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
|
||||
def list_all_imports(site, status \\ nil) do
|
||||
imports =
|
||||
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|
||||
|> maybe_filter_by_status(status)
|
||||
|> Repo.all()
|
||||
|
||||
if site.imported_data && not Enum.any?(imports, & &1.legacy) do
|
||||
imports ++ [SiteImport.from_legacy(site.imported_data)]
|
||||
else
|
||||
imports
|
||||
end
|
||||
end
|
||||
|
||||
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
|
||||
def other_imports_in_progress?(site_import) do
|
||||
Repo.exists?(
|
||||
from(i in SiteImport,
|
||||
where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
|
||||
where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
defp maybe_filter_by_status(query, nil), do: query
|
||||
|
||||
defp maybe_filter_by_status(query, status) do
|
||||
where(query, [i], i.status == ^status)
|
||||
end
|
||||
|
||||
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
|
||||
def list_complete_import_ids(site) do
|
||||
ids =
|
||||
from(i in SiteImport,
|
||||
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
|
||||
select: {i.legacy, i.id},
|
||||
limit: @max_complete_imports
|
||||
)
|
||||
|> Repo.all()
|
||||
|
||||
has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)
|
||||
ids = Enum.map(ids, &elem(&1, 1))
|
||||
|
||||
# account for legacy imports as well
|
||||
if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do
|
||||
[0 | ids]
|
||||
else
|
||||
ids
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_imports_date_range(Site.t()) :: %{
|
||||
start_date: Date.t() | nil,
|
||||
end_date: Date.t() | nil
|
||||
}
|
||||
def get_imports_date_range(site) do
|
||||
dates =
|
||||
from(i in SiteImport,
|
||||
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
|
||||
select: %{start_date: min(i.start_date), end_date: max(i.end_date)}
|
||||
)
|
||||
|> Repo.one()
|
||||
|
||||
dates = dates || %{start_date: nil, end_date: nil}
|
||||
|
||||
if site.imported_data && site.imported_data.status == "ok" do
|
||||
start_date =
|
||||
[dates.start_date, site.imported_data.start_date]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.min(Date, fn -> nil end)
|
||||
|
||||
end_date =
|
||||
[dates.end_date, site.imported_data.end_date]
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.max(Date, fn -> nil end)
|
||||
|
||||
%{start_date: start_date, end_date: end_date}
|
||||
else
|
||||
dates
|
||||
end
|
||||
end
|
||||
|
||||
@spec delete_imports_for_site(Site.t()) :: :ok
|
||||
def delete_imports_for_site(site) do
|
||||
Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) ::
|
||||
{:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window}
|
||||
def check_dates(_site, nil, _end_date), do: {:error, :no_data}
|
||||
|
||||
def check_dates(site, start_date, end_date) do
|
||||
cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
|
||||
end_date = Enum.min([end_date, cutoff_date], Date)
|
||||
|
||||
with true <- Date.diff(end_date, start_date) >= 2,
|
||||
[_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do
|
||||
longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first))
|
||||
{:ok, longest.first, longest.last}
|
||||
else
|
||||
_ -> {:error, :no_time_window}
|
||||
end
|
||||
end
|
||||
|
||||
defp find_free_ranges(start_date, end_date, site) do
|
||||
occupied_ranges =
|
||||
site
|
||||
|> Imported.list_all_imports(Imported.SiteImport.completed())
|
||||
|> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2))
|
||||
|> Enum.map(&Date.range(&1.start_date, &1.end_date))
|
||||
|
||||
Date.range(start_date, end_date)
|
||||
|> free_ranges(start_date, occupied_ranges, [])
|
||||
end
|
||||
|
||||
# This function recursively finds open ranges that are not yet occupied
|
||||
# by existing imported data. The idea is that we keep moving a dynamic
|
||||
# date index `d` from start until the end of `imported_range`, hopping
|
||||
# over each occupied range, and capturing the open ranges step-by-step
|
||||
# in the `result` array.
|
||||
defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do
|
||||
cond do
|
||||
Date.diff(occupied_range.last, d) <= 0 ->
|
||||
free_ranges(import_range, d, rest_of_occupied_ranges, result)
|
||||
|
||||
in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 ->
|
||||
d = occupied_range.last
|
||||
free_ranges(import_range, d, rest_of_occupied_ranges, result)
|
||||
|
||||
true ->
|
||||
free_range = Date.range(d, occupied_range.first)
|
||||
result = result ++ [free_range]
|
||||
d = occupied_range.last
|
||||
free_ranges(import_range, d, rest_of_occupied_ranges, result)
|
||||
end
|
||||
end
|
||||
|
||||
defp free_ranges(import_range, d, [], result) do
|
||||
if Date.diff(import_range.last, d) < 2 do
|
||||
result
|
||||
else
|
||||
result ++ [Date.range(d, import_range.last)]
|
||||
end
|
||||
end
|
||||
|
||||
defp in_range?(date, range) do
|
||||
Date.before?(range.first, date) && Date.after?(range.last, date)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,229 @@
|
||||
defmodule Plausible.Imported.CSVImporter do
|
||||
@moduledoc """
|
||||
CSV importer from S3 that uses ClickHouse [s3 table function.](https://clickhouse.com/docs/en/sql-reference/table-functions/s3)
|
||||
"""
|
||||
|
||||
use Plausible.Imported.Importer
|
||||
|
||||
@impl true
|
||||
def name(), do: :csv
|
||||
|
||||
@impl true
|
||||
def label(), do: "CSV"
|
||||
|
||||
# NOTE: change it once CSV import is implemented
|
||||
@impl true
|
||||
def email_template(), do: "google_analytics_import.html"
|
||||
|
||||
@impl true
|
||||
def parse_args(%{"uploads" => uploads}), do: [uploads: uploads]
|
||||
|
||||
@impl true
|
||||
def import_data(site_import, opts) do
|
||||
%{
|
||||
id: import_id,
|
||||
site_id: site_id,
|
||||
start_date: start_date,
|
||||
end_date: end_date
|
||||
} = site_import
|
||||
|
||||
uploads = Keyword.fetch!(opts, :uploads)
|
||||
|
||||
%{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} =
|
||||
Plausible.S3.import_clickhouse_credentials()
|
||||
|
||||
{:ok, ch} =
|
||||
Plausible.IngestRepo.config()
|
||||
|> Keyword.replace!(:pool_size, 1)
|
||||
|> Ch.start_link()
|
||||
|
||||
Enum.each(uploads, fn upload ->
|
||||
%{"filename" => filename, "s3_url" => s3_url} = upload
|
||||
|
||||
{table, _, _} = parse_filename!(filename)
|
||||
s3_structure = input_structure!(table)
|
||||
|
||||
s3_structure_cols_expr =
|
||||
s3_structure
|
||||
|> String.split(",", trim: true)
|
||||
|> Enum.map_join(", ", fn kv ->
|
||||
[col, _type] = String.split(kv)
|
||||
col
|
||||
end)
|
||||
|
||||
statement =
|
||||
"""
|
||||
INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \
|
||||
SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \
|
||||
FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \
|
||||
WHERE date >= {start_date:Date} AND date <= {end_date:Date}\
|
||||
"""
|
||||
|
||||
params =
|
||||
%{
|
||||
"table" => table,
|
||||
"site_id" => site_id,
|
||||
"import_id" => import_id,
|
||||
"s3_url" => s3_url,
|
||||
"s3_access_key_id" => s3_access_key_id,
|
||||
"s3_secret_access_key" => s3_secret_access_key,
|
||||
"s3_format" => "CSVWithNames",
|
||||
"s3_structure" => s3_structure,
|
||||
"start_date" => start_date,
|
||||
"end_date" => end_date
|
||||
}
|
||||
|
||||
Ch.query!(ch, statement, params, timeout: :infinity)
|
||||
end)
|
||||
rescue
|
||||
# we are cancelling on any argument or ClickHouse errors
|
||||
e in [ArgumentError, Ch.Error] ->
|
||||
{:error, Exception.message(e)}
|
||||
end
|
||||
|
||||
input_structures = %{
|
||||
"imported_browsers" =>
|
||||
"date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_devices" =>
|
||||
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_entry_pages" =>
|
||||
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_exit_pages" =>
|
||||
"date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_locations" =>
|
||||
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_operating_systems" =>
|
||||
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
|
||||
"imported_pages" =>
|
||||
"date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
|
||||
"imported_sources" =>
|
||||
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
|
||||
"imported_visitors" =>
|
||||
"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
|
||||
}
|
||||
|
||||
@doc """
|
||||
Extracts min/max date range from a list of uploads.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> date_range([
|
||||
...> %{"filename" => "imported_devices_20190101_20210101.csv"},
|
||||
...> "imported_pages_20200101_20220101.csv"
|
||||
...> ])
|
||||
Date.range(~D[2019-01-01], ~D[2022-01-01])
|
||||
|
||||
iex> date_range([])
|
||||
nil
|
||||
|
||||
"""
|
||||
@spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
|
||||
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
|
||||
def date_range([]), do: nil
|
||||
|
||||
defp date_range([upload | uploads], prev_start_date, prev_end_date) do
|
||||
filename =
|
||||
case upload do
|
||||
%{"filename" => filename} -> filename
|
||||
filename when is_binary(filename) -> filename
|
||||
end
|
||||
|
||||
{_table, start_date, end_date} = parse_filename!(filename)
|
||||
|
||||
start_date =
|
||||
if prev_start_date do
|
||||
Enum.min([start_date, prev_start_date], Date)
|
||||
else
|
||||
start_date
|
||||
end
|
||||
|
||||
end_date =
|
||||
if prev_end_date do
|
||||
Enum.max([end_date, prev_end_date], Date)
|
||||
else
|
||||
end_date
|
||||
end
|
||||
|
||||
date_range(uploads, start_date, end_date)
|
||||
end
|
||||
|
||||
defp date_range([], first, last), do: Date.range(first, last)
|
||||
|
||||
@spec parse_date!(String.t()) :: Date.t()
|
||||
defp parse_date!(date) do
|
||||
date |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Extracts table name and min/max dates from the filename.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> parse_filename!("my_data.csv")
|
||||
** (ArgumentError) invalid filename
|
||||
|
||||
iex> parse_filename!("imported_devices_00010101_20250101.csv")
|
||||
{"imported_devices", ~D[0001-01-01], ~D[2025-01-01]}
|
||||
|
||||
"""
|
||||
@spec parse_filename!(String.t()) ::
|
||||
{table :: String.t(), start_date :: Date.t(), end_date :: Date.t()}
|
||||
def parse_filename!(filename)
|
||||
|
||||
for {table, input_structure} <- input_structures do
|
||||
defp input_structure!(unquote(table)), do: unquote(input_structure)
|
||||
|
||||
def parse_filename!(
|
||||
<<unquote(table)::bytes, ?_, start_date::8-bytes, ?_, end_date::8-bytes, ".csv">>
|
||||
) do
|
||||
{unquote(table), parse_date!(start_date), parse_date!(end_date)}
|
||||
end
|
||||
end
|
||||
|
||||
def parse_filename!(_filename) do
|
||||
raise ArgumentError, "invalid filename"
|
||||
end
|
||||
|
||||
@doc """
|
||||
Checks if the provided filename conforms to the expected format.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> valid_filename?("my_data.csv")
|
||||
false
|
||||
|
||||
iex> valid_filename?("imported_devices_00010101_20250101.csv")
|
||||
true
|
||||
|
||||
"""
|
||||
@spec valid_filename?(String.t()) :: boolean
|
||||
def valid_filename?(filename) do
|
||||
try do
|
||||
parse_filename!(filename)
|
||||
else
|
||||
_ -> true
|
||||
rescue
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Extracts the table name from the provided filename.
|
||||
|
||||
Raises if the filename doesn't conform to the expected format.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> extract_table("my_data.csv")
|
||||
** (ArgumentError) invalid filename
|
||||
|
||||
iex> extract_table("imported_devices_00010101_20250101.csv")
|
||||
"imported_devices"
|
||||
|
||||
"""
|
||||
@spec extract_table(String.t()) :: String.t()
|
||||
def extract_table(filename) do
|
||||
{table, _start_date, _end_date} = parse_filename!(filename)
|
||||
table
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,276 @@
|
||||
defmodule Plausible.Imported.GoogleAnalytics4 do
|
||||
@moduledoc """
|
||||
Import implementation for Google Analytics 4.
|
||||
"""
|
||||
|
||||
use Plausible.Imported.Importer
|
||||
|
||||
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
|
||||
|
||||
@impl true
|
||||
def name(), do: :google_analytics_4
|
||||
|
||||
@impl true
|
||||
def label(), do: "Google Analytics 4"
|
||||
|
||||
@impl true
|
||||
def email_template(), do: "google_analytics_import.html"
|
||||
|
||||
@impl true
|
||||
def parse_args(
|
||||
%{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
|
||||
) do
|
||||
start_date = Date.from_iso8601!(start_date)
|
||||
end_date = Date.from_iso8601!(end_date)
|
||||
date_range = Date.range(start_date, end_date)
|
||||
|
||||
auth = {
|
||||
Map.fetch!(args, "access_token"),
|
||||
Map.fetch!(args, "refresh_token"),
|
||||
Map.fetch!(args, "token_expires_at")
|
||||
}
|
||||
|
||||
[
|
||||
property: property,
|
||||
date_range: date_range,
|
||||
auth: auth
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Imports stats from a Google Analytics 4 property to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics 4 reports which are then passed in batches
|
||||
to Clickhouse by the `Plausible.Imported.Buffer` process.
|
||||
"""
|
||||
@impl true
|
||||
def import_data(site_import, opts) do
|
||||
date_range = Keyword.fetch!(opts, :date_range)
|
||||
property = Keyword.fetch!(opts, :property)
|
||||
auth = Keyword.fetch!(opts, :auth)
|
||||
flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
|
||||
|
||||
{:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
|
||||
|
||||
persist_fn = fn table, rows ->
|
||||
records = from_report(rows, site_import.site_id, site_import.id, table)
|
||||
Plausible.Imported.Buffer.insert_many(buffer, table, records)
|
||||
end
|
||||
|
||||
try do
|
||||
Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
|
||||
after
|
||||
Plausible.Imported.Buffer.flush(buffer)
|
||||
Plausible.Imported.Buffer.stop(buffer)
|
||||
end
|
||||
end
|
||||
|
||||
def from_report(nil, _site_id, _import_id, _metric), do: nil
|
||||
|
||||
def from_report(data, site_id, import_id, table) do
|
||||
Enum.reduce(data, [], fn row, acc ->
|
||||
if Map.get(row.dimensions, "date") in @missing_values do
|
||||
acc
|
||||
else
|
||||
[new_from_report(site_id, import_id, table, row) | acc]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp parse_number(nr) do
|
||||
{float, ""} = Float.parse(nr)
|
||||
round(float)
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_visitors", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_sources", row) do
|
||||
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
|
||||
|
||||
referrer =
|
||||
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
|
||||
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
|
||||
end
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
|
||||
referrer: referrer,
|
||||
# Only `source` exists in GA4 API
|
||||
utm_source: nil,
|
||||
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
|
||||
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
|
||||
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
|
||||
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
|
||||
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# NOTE: no exits metric in GA4 API currently
|
||||
exits: 0,
|
||||
time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
entry_page: row.dimensions |> Map.fetch!("landingPage"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
# NOTE: no exit pages metrics in GA4 API available for now
|
||||
# defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
|
||||
# %{
|
||||
# site_id: site_id,
|
||||
# import_id: import_id,
|
||||
# date: get_date(row),
|
||||
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
|
||||
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
# }
|
||||
# end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_locations", row) do
|
||||
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
|
||||
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
|
||||
city_data = Location.get_city(city_name, country_code)
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
country: country_code,
|
||||
region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
|
||||
city: city_data && city_data.id,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_devices", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@browser_google_to_plausible %{
|
||||
"User-Agent:Opera" => "Opera",
|
||||
"Mozilla Compatible Agent" => "Mobile App",
|
||||
"Android Webview" => "Mobile App",
|
||||
"Android Browser" => "Mobile App",
|
||||
"Safari (in-app)" => "Mobile App",
|
||||
"User-Agent: Mozilla" => "Firefox",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_browsers", row) do
|
||||
browser = Map.fetch!(row.dimensions, "browser")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
browser: Map.get(@browser_google_to_plausible, browser, browser),
|
||||
# Does not exist in GA4 API
|
||||
browser_version: nil,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@os_google_to_plausible %{
|
||||
"Macintosh" => "Mac",
|
||||
"Linux" => "GNU/Linux",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
|
||||
os = Map.fetch!(row.dimensions, "operatingSystem")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
operating_system: Map.get(@os_google_to_plausible, os, os),
|
||||
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp get_date(%{dimensions: %{"date" => date}}) do
|
||||
date
|
||||
|> Timex.parse!("%Y%m%d", :strftime)
|
||||
|> NaiveDateTime.to_date()
|
||||
end
|
||||
|
||||
defp default_if_missing(value, default \\ nil)
|
||||
defp default_if_missing(value, default) when value in @missing_values, do: default
|
||||
defp default_if_missing(value, _default), do: value
|
||||
|
||||
defp parse_referrer(nil), do: nil
|
||||
defp parse_referrer("(direct)"), do: nil
|
||||
defp parse_referrer("google"), do: "Google"
|
||||
defp parse_referrer("bing"), do: "Bing"
|
||||
defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
|
||||
|
||||
defp parse_referrer(ref) do
|
||||
RefInspector.parse("https://" <> ref)
|
||||
|> PlausibleWeb.RefInspector.parse()
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,247 @@
|
||||
defmodule Plausible.Imported.Importer do
|
||||
@moduledoc """
|
||||
Behaviour that should be implemented for each import source.
|
||||
|
||||
All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`
|
||||
Oban worker. Each import source must define a module conforming `Importer` behaviour.
|
||||
|
||||
The callbacks that need to be implemented:
|
||||
|
||||
* `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.
|
||||
* `label/0` - Descriptive, display friendly name of the source.
|
||||
Example: "Google Analytics".
|
||||
* `email_template/0` - Name of the email template to use for notifications in
|
||||
`PlausibleWeb.Email` (`import_success` and `import_failure`). The template
|
||||
should have content customized for a particular source.
|
||||
* `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever
|
||||
options were passed to `new_import/3` will be present in the input map with string
|
||||
keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`
|
||||
is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.
|
||||
The expectation is parsing the map values producing a keyword list of options to
|
||||
pass to `import_data/2`.
|
||||
* `import_data/2` - Receives site import struct and options produced by `parse_args/1`.
|
||||
This is where all the import processing is done. The way the import is implemented
|
||||
is entirely arbitrary except the requirement that the process as a whole must
|
||||
by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`
|
||||
on successful import or `{:error, ...}` on failure. The map in success tuple is
|
||||
used for updating site import struct and is passed to `on_success/2` callback.
|
||||
Please note that error tuple should be only returned on errors that can't be
|
||||
recovered from. For transient errors, the import should throw an exception or
|
||||
simply crash.
|
||||
* `before_start/1` - Optional callback run right before scheduling import job. It's
|
||||
expected to either return `:ok` for the import to proceed or `{:error, ...}` tuple,
|
||||
which will be returned from `new_import/3` call.
|
||||
* `on_success/2` - Optional callback run once site import is completed. Receives map
|
||||
returned from `import_data/2`. Expected to always return `:ok`.
|
||||
* `on_failure/1` - Optional callback run when import job fails permanently.
|
||||
|
||||
All sources must be added to the list in `Plausible.Imported.ImportSources`.
|
||||
|
||||
In order to schedule a new import job using a given source, respective importer's
|
||||
`new_import/3` function must be called. It accepts site, user who is doing the import
|
||||
and any options necessary to carry out the import.
|
||||
|
||||
There's an expectation that `start_date` and `end_date` are provided either as options
|
||||
passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters
|
||||
are not provided, the import will eventually crash. These parameters define time range
|
||||
of imported data which is in turn used for efficient querying.
|
||||
|
||||
Logic running inside `import_data/2` is expected to populated all `imported_*` tables
|
||||
in ClickHouse with `import_id` column set to site import's ID.
|
||||
|
||||
Managing any configuration or authentication prior to running import is outside of
|
||||
scope of importer logic and is expected to be implemented separately.
|
||||
|
||||
## Running import fully synchronously
|
||||
|
||||
In case it's necessary to run the whole import job fully synchronously, the
|
||||
`Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message
|
||||
on completion, failure or transient failure of the import.
|
||||
|
||||
A basic usage scenario looks like this:
|
||||
|
||||
```elixir
|
||||
{:ok, job} = Plausible.Imported.NoopImporter.new_import(
|
||||
site,
|
||||
user,
|
||||
start_date: ~D[2005-01-01],
|
||||
end_date: Date.utc_today(),
|
||||
# this option is necessary to setup the calling process as listener
|
||||
listen?: true
|
||||
)
|
||||
|
||||
import_id = job.args[:import_id]
|
||||
|
||||
receive do
|
||||
{:notification, :analytics_imports_jobs, %{"complete" => ^import_id}} ->
|
||||
IO.puts("Job completed")
|
||||
|
||||
{:notification, :analytics_imports_jobs, %{"transient_fail" => ^import_id}} ->
|
||||
IO.puts("Job failed transiently")
|
||||
|
||||
{:notification, :analytics_imports_jobs, %{"fail" => ^import_id}} ->
|
||||
IO.puts("Job failed permanently")
|
||||
after
|
||||
15_000 ->
|
||||
IO.puts("Job didn't finish in 15 seconds")
|
||||
end
|
||||
```
|
||||
|
||||
In a more realistic scenario, job scheduling will be done inside a GenServer process
|
||||
like LiveView, where notifications can be listened for via `handle_info/2`.
|
||||
"""
|
||||
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Imported.SiteImport
|
||||
alias Plausible.Repo
|
||||
|
||||
@callback name() :: atom()
|
||||
@callback label() :: String.t()
|
||||
@callback email_template() :: String.t()
|
||||
@callback parse_args(map()) :: Keyword.t()
|
||||
@callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
|
||||
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
|
||||
@callback on_success(SiteImport.t(), map()) :: :ok
|
||||
@callback on_failure(SiteImport.t()) :: :ok
|
||||
|
||||
defmacro __using__(_opts) do
|
||||
quote do
|
||||
@behaviour Plausible.Imported.Importer
|
||||
|
||||
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
|
||||
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress}
|
||||
def new_import(site, user, opts) do
|
||||
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
|
||||
end
|
||||
|
||||
@doc false
|
||||
@spec run_import(SiteImport.t(), map()) :: :ok | {:error, :any}
|
||||
def run_import(site_import, args) do
|
||||
Plausible.Imported.Importer.run_import(
|
||||
site_import,
|
||||
args,
|
||||
&parse_args/1,
|
||||
&import_data/2,
|
||||
&on_success/2
|
||||
)
|
||||
end
|
||||
|
||||
@doc false
|
||||
@spec mark_failed(SiteImport.t()) :: SiteImport.t()
|
||||
def mark_failed(site_import) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.fail_changeset()
|
||||
|> Repo.update!()
|
||||
|
||||
:ok = on_failure(site_import)
|
||||
|
||||
site_import
|
||||
end
|
||||
|
||||
@impl true
|
||||
def before_start(_site_import), do: :ok
|
||||
|
||||
@impl true
|
||||
def on_success(_site_import, _extra_data), do: :ok
|
||||
|
||||
@impl true
|
||||
def on_failure(_site_import), do: :ok
|
||||
|
||||
defoverridable before_start: 1, on_success: 2, on_failure: 1
|
||||
end
|
||||
end
|
||||
|
||||
@doc false
|
||||
def new_import(source, site, user, opts, before_start_fun) do
|
||||
import_params =
|
||||
opts
|
||||
|> Keyword.take([:label, :start_date, :end_date, :legacy])
|
||||
|> Keyword.put(:source, source)
|
||||
|> Map.new()
|
||||
|
||||
Repo.transaction(fn ->
|
||||
result =
|
||||
site
|
||||
|> SiteImport.create_changeset(user, import_params)
|
||||
|> Repo.insert()
|
||||
|
||||
with {:ok, site_import} <- result,
|
||||
:ok <- before_start_fun.(site_import),
|
||||
{:ok, job} <- schedule_job(site_import, opts) do
|
||||
job
|
||||
else
|
||||
{:error, error} ->
|
||||
Repo.rollback(error)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.start_changeset()
|
||||
|> Repo.update!()
|
||||
|> Repo.preload(:site)
|
||||
|
||||
import_opts = parse_args_fun.(args)
|
||||
|
||||
case import_fun.(site_import, import_opts) do
|
||||
:ok ->
|
||||
{:ok, mark_complete(site_import, %{}, on_success_fun)}
|
||||
|
||||
{:ok, extra_data} ->
|
||||
{:ok, mark_complete(site_import, extra_data, on_success_fun)}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@oban_channel :analytics_imports_jobs
|
||||
|
||||
@doc false
|
||||
def notify(site_import, event) do
|
||||
Oban.Notifier.notify(Oban, @oban_channel, %{event => site_import.id})
|
||||
end
|
||||
|
||||
@doc """
|
||||
Allows to explicitly start listening for importer job notifications.
|
||||
|
||||
Listener must explicitly filter out a subset of imports that apply to the given context.
|
||||
"""
|
||||
@spec listen() :: :ok
|
||||
def listen() do
|
||||
:ok = Oban.Notifier.listen([@oban_channel])
|
||||
end
|
||||
|
||||
defp schedule_job(site_import, opts) do
|
||||
{listen?, opts} = Keyword.pop(opts, :listen?, false)
|
||||
|
||||
if listen? do
|
||||
:ok = listen()
|
||||
end
|
||||
|
||||
if not Imported.other_imports_in_progress?(site_import) do
|
||||
opts
|
||||
|> Keyword.put(:import_id, site_import.id)
|
||||
|> Map.new()
|
||||
|> Plausible.Workers.ImportAnalytics.new()
|
||||
|> Oban.insert()
|
||||
else
|
||||
{:error, :import_in_progress}
|
||||
end
|
||||
end
|
||||
|
||||
defp mark_complete(site_import, extra_data, on_success_fun) do
|
||||
site_import =
|
||||
site_import
|
||||
|> SiteImport.complete_changeset(extra_data)
|
||||
|> Repo.update!()
|
||||
|
||||
:ok = on_success_fun.(site_import, extra_data)
|
||||
|
||||
site_import
|
||||
end
|
||||
end
|
||||
@ -1,13 +0,0 @@
|
||||
defmodule Plausible.Imported do
|
||||
@tables ~w(
|
||||
imported_visitors imported_sources imported_pages imported_entry_pages
|
||||
imported_exit_pages imported_locations imported_devices imported_browsers
|
||||
imported_operating_systems
|
||||
)
|
||||
@spec tables() :: [String.t()]
|
||||
def tables, do: @tables
|
||||
|
||||
def forget(site) do
|
||||
Plausible.Purge.delete_imported_stats!(site)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,95 @@
|
||||
defmodule Plausible.Imported.SiteImport do
|
||||
@moduledoc """
|
||||
Site import schema.
|
||||
"""
|
||||
|
||||
use Ecto.Schema
|
||||
|
||||
import Ecto.Changeset
|
||||
|
||||
alias Plausible.Auth.User
|
||||
alias Plausible.Imported.ImportSources
|
||||
alias Plausible.Site
|
||||
|
||||
@statuses [:pending, :importing, :completed, :failed]
|
||||
|
||||
@type t() :: %__MODULE__{}
|
||||
|
||||
schema "site_imports" do
|
||||
field :start_date, :date
|
||||
field :end_date, :date
|
||||
field :label, :string
|
||||
field :source, Ecto.Enum, values: ImportSources.names()
|
||||
field :status, Ecto.Enum, values: @statuses
|
||||
field :legacy, :boolean, default: false
|
||||
|
||||
belongs_to :site, Site
|
||||
belongs_to :imported_by, User
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
for status <- @statuses do
|
||||
defmacro unquote(status)(), do: unquote(status)
|
||||
end
|
||||
|
||||
@spec label(t() | Site.ImportedData.t()) :: String.t()
|
||||
def label(%__MODULE__{source: source, label: label}) do
|
||||
build_label(ImportSources.by_name(source).label(), label)
|
||||
end
|
||||
|
||||
# NOTE: this is necessary for backwards compatibility
|
||||
# with legacy imports
|
||||
def label(%Site.ImportedData{source: source}), do: build_label(source, nil)
|
||||
|
||||
@spec from_legacy(Site.ImportedData.t()) :: t()
|
||||
def from_legacy(%Site.ImportedData{} = data) do
|
||||
status =
|
||||
case data.status do
|
||||
"ok" -> completed()
|
||||
"error" -> failed()
|
||||
_ -> importing()
|
||||
end
|
||||
|
||||
%__MODULE__{
|
||||
id: 0,
|
||||
legacy: true,
|
||||
start_date: data.start_date,
|
||||
end_date: data.end_date,
|
||||
source: :universal_analytics,
|
||||
status: status
|
||||
}
|
||||
end
|
||||
|
||||
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
|
||||
def create_changeset(site, user, params) do
|
||||
%__MODULE__{}
|
||||
|> cast(params, [:label, :source, :start_date, :end_date, :legacy])
|
||||
|> validate_required([:source])
|
||||
|> put_assoc(:site, site)
|
||||
|> put_assoc(:imported_by, user)
|
||||
|> put_change(:status, pending())
|
||||
end
|
||||
|
||||
@spec start_changeset(t()) :: Ecto.Changeset.t()
|
||||
def start_changeset(site_import) do
|
||||
site_import
|
||||
|> change(status: importing())
|
||||
end
|
||||
|
||||
@spec complete_changeset(t(), map()) :: Ecto.Changeset.t()
|
||||
def complete_changeset(site_import, params \\ %{}) do
|
||||
site_import
|
||||
|> cast(params, [:start_date, :end_date])
|
||||
|> put_change(:status, completed())
|
||||
|> validate_required([:start_date, :end_date])
|
||||
end
|
||||
|
||||
@spec fail_changeset(t()) :: Ecto.Changeset.t()
|
||||
def fail_changeset(site_import) do
|
||||
change(site_import, status: failed())
|
||||
end
|
||||
|
||||
defp build_label(source, nil), do: source
|
||||
defp build_label(source, label), do: "#{source} (#{label})"
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue