Compare commits
105 Commits
drop-event
...
master
| Author | SHA1 | Date |
|---|---|---|
|
|
a7603c9e49 | 3 years ago |
|
|
cffff0340c | 3 years ago |
|
|
33eed9d7db | 3 years ago |
|
|
e5b7f1afd0 | 3 years ago |
|
|
f966419a4a | 3 years ago |
|
|
f9f0407d68 | 3 years ago |
|
|
e5b56dbe62 | 3 years ago |
|
|
3115c6e7a8 | 3 years ago |
|
|
6af80dd246 | 3 years ago |
|
|
e6d83e946f | 3 years ago |
|
|
bd73fc8266 | 3 years ago |
|
|
2ba4988c95 | 3 years ago |
|
|
1f778e0c11 | 3 years ago |
|
|
9f27fa303c | 3 years ago |
|
|
23a3699dd7 | 3 years ago |
|
|
71fe541359 | 3 years ago |
|
|
5bf59d1d8a | 3 years ago |
|
|
c263df5805 | 3 years ago |
|
|
b31433a7bf | 3 years ago |
|
|
edf70d14b6 | 3 years ago |
|
|
7523abe93e | 3 years ago |
|
|
604bf88451 | 3 years ago |
|
|
4af7019011 | 3 years ago |
|
|
2fae0146a4 | 3 years ago |
|
|
9989ce6927 | 3 years ago |
|
|
53f94a9f82 | 3 years ago |
|
|
ba5b80a8c0 | 3 years ago |
|
|
1d017e86a1 | 3 years ago |
|
|
52c226c428 | 3 years ago |
|
|
fb61f0b425 | 3 years ago |
|
|
d6e1e8bebd | 3 years ago |
|
|
561dcd821e | 3 years ago |
|
|
fd879eeb16 | 3 years ago |
|
|
8992c8ee07 | 3 years ago |
|
|
c32779a3e5 | 3 years ago |
|
|
d6e81670e4 | 3 years ago |
|
|
5f9465614b | 3 years ago |
|
|
32ab138301 | 3 years ago |
|
|
c219652dae | 3 years ago |
|
|
253fb5d67d | 3 years ago |
|
|
1909743b90 | 3 years ago |
|
|
64f35bf8f6 | 3 years ago |
|
|
279e89c693 | 3 years ago |
|
|
4242b52be4 | 3 years ago |
|
|
dfcc8d794a | 3 years ago |
|
|
02d2256483 | 3 years ago |
|
|
07b714a143 | 3 years ago |
|
|
c08aa29ad8 | 3 years ago |
|
|
0a0c7ce165 | 3 years ago |
|
|
8c0e94f72c | 3 years ago |
|
|
4ae00540dc | 3 years ago |
|
|
5e74b1cf74 | 3 years ago |
|
|
d3586a81e3 | 3 years ago |
|
|
59afa20955 | 3 years ago |
|
|
e8f3946dde | 3 years ago |
|
|
4d7d88cfec | 3 years ago |
|
|
f2350b5165 | 3 years ago |
|
|
5a3072ca21 | 3 years ago |
|
|
6eef32a8ff | 3 years ago |
|
|
7641c66a2b | 3 years ago |
|
|
dea9287d74 | 3 years ago |
|
|
c5881cdc6d | 3 years ago |
|
|
ca7edd47cb | 3 years ago |
|
|
437a3350ff | 3 years ago |
|
|
bb8a272a36 | 3 years ago |
|
|
b68e3aa6a2 | 3 years ago |
|
|
83a46fb47e | 3 years ago |
|
|
11368e5143 | 3 years ago |
|
|
a9d3c03782 | 3 years ago |
|
|
26d41ddbb9 | 3 years ago |
|
|
0cdba7d407 | 3 years ago |
|
|
c6d98397a8 | 3 years ago |
|
|
c60a2faee4 | 3 years ago |
|
|
70d21f1b69 | 3 years ago |
|
|
8d977e0f76 | 3 years ago |
|
|
fecd665c19 | 3 years ago |
|
|
d5048fd6b4 | 3 years ago |
|
|
5eb9d724e5 | 3 years ago |
|
|
ea38b45685 | 3 years ago |
|
|
39aa81a16f | 3 years ago |
|
|
fdbe4cc0a4 | 3 years ago |
|
|
0fe5682f91 | 3 years ago |
|
|
316148e788 | 3 years ago |
|
|
f3423aefec | 3 years ago |
|
|
31cf3e54f8 | 3 years ago |
|
|
f1b6a672d4 | 3 years ago |
|
|
518cdb3307 | 3 years ago |
|
|
b7b5dcf4c9 | 3 years ago |
|
|
8a8df685a7 | 3 years ago |
|
|
dd428430f5 | 3 years ago |
|
|
cc867396b2 | 3 years ago |
|
|
bea8ceeffb | 3 years ago |
|
|
c822f8de1e | 3 years ago |
|
|
c6e06359c6 | 3 years ago |
|
|
19ce62db61 | 3 years ago |
|
|
a1fedb751b | 3 years ago |
|
|
0f0f604d2e | 3 years ago |
|
|
bdaf80c05d | 3 years ago |
|
|
acd4f33c8e | 3 years ago |
|
|
bca617d0bf | 3 years ago |
|
|
52f584efa9 | 3 years ago |
|
|
8ba2b934b3 | 3 years ago |
|
|
d74b1d5e60 | 3 years ago |
|
|
6035618213 | 3 years ago |
|
|
23b9032148 | 3 years ago |
@ -0,0 +1,73 @@
|
||||
# This file excludes paths from the Docker build context.
|
||||
#
|
||||
# By default, Docker's build context includes all files (and folders) in the
|
||||
# current directory. Even if a file isn't copied into the container it is still sent to
|
||||
# the Docker daemon.
|
||||
#
|
||||
# There are multiple reasons to exclude files from the build context:
|
||||
#
|
||||
# 1. Prevent nested folders from being copied into the container (ex: exclude
|
||||
# /assets/node_modules when copying /assets)
|
||||
# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc)
|
||||
# 3. Avoid sending files containing sensitive information
|
||||
#
|
||||
# More information on using .dockerignore is available here:
|
||||
# https://docs.docker.com/engine/reference/builder/#dockerignore-file
|
||||
|
||||
.dockerignore
|
||||
|
||||
# Ignore git, but keep git HEAD and refs to access current commit hash if needed:
|
||||
#
|
||||
# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat
|
||||
# d0b8727759e1e0e7aa3d41707d12376e373d5ecc
|
||||
.git
|
||||
!.git/HEAD
|
||||
!.git/refs
|
||||
|
||||
# Common development/test artifacts
|
||||
/cover/
|
||||
/doc/
|
||||
/test/
|
||||
/tmp/
|
||||
.elixir_ls
|
||||
plausible-report.xml
|
||||
.env
|
||||
.idea
|
||||
*.iml
|
||||
*.log
|
||||
*.code-workspace
|
||||
.vscode
|
||||
|
||||
# Mix artifacts
|
||||
/_build/
|
||||
/deps/
|
||||
*.ez
|
||||
|
||||
# Generated on crash by the VM
|
||||
erl_crash.dump
|
||||
|
||||
# If NPM crashes, it generates a log, let's ignore it too.
|
||||
npm-debug.log
|
||||
|
||||
# Static artifacts - These should be fetched and built inside the Docker image
|
||||
/assets/node_modules/
|
||||
/tracker/node_modules/
|
||||
/priv/static/cache_manifest.json
|
||||
/priv/static/css
|
||||
/priv/static/js
|
||||
/priv/version.json
|
||||
|
||||
# Auto-generated tracker files
|
||||
/priv/tracker/js/*.js
|
||||
|
||||
# Dializer
|
||||
/priv/plts/*.plt
|
||||
/priv/plts/*.plt.hash
|
||||
|
||||
# Geolocation databases
|
||||
/priv/geodb/*.mmdb
|
||||
/priv/geodb/*.mmdb.gz
|
||||
|
||||
# Docker volumes
|
||||
.clickhouse_db_vol*
|
||||
plausible_db*
|
||||
@ -0,0 +1,67 @@
|
||||
name: Build Private Images GHCR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, stable]
|
||||
tags: ['r*']
|
||||
pull_request:
|
||||
types: [synchronize, labeled]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') }}
|
||||
runs-on: buildjet-16vcpu-ubuntu-2204
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5.0.0
|
||||
with:
|
||||
images: ghcr.io/plausible/analytics/ee
|
||||
tags: |
|
||||
type=ref,event=pr
|
||||
type=ref,event=branch
|
||||
type=ref,event=tag
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
MIX_ENV=prod
|
||||
BUILD_METADATA=${{ steps.meta.outputs.json }}
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Notify team on failure
|
||||
if: ${{ failure() }}
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
|
||||
method: 'POST'
|
||||
customHeaders: '{"Content-Type": "application/json"}'
|
||||
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-private-images.yml\">Build failed</a>"}'
|
||||
@ -0,0 +1,68 @@
|
||||
name: Build Public Images GHCR
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: buildjet-16vcpu-ubuntu-2204
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/plausible/community-edition
|
||||
tags: |
|
||||
type=semver,pattern={{version}},prefix=v
|
||||
type=semver,pattern={{major}}.{{minor}},prefix=v
|
||||
type=semver,pattern={{major}},prefix=v
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64, linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
build-args: |
|
||||
MIX_ENV=small
|
||||
BUILD_METADATA=${{ steps.meta.outputs.json }}
|
||||
ERL_FLAGS=+JPperf true
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Notify team on failure
|
||||
if: ${{ failure() }}
|
||||
uses: fjogeleit/http-request-action@v1
|
||||
with:
|
||||
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
|
||||
method: 'POST'
|
||||
customHeaders: '{"Content-Type": "application/json"}'
|
||||
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-public-images.yml\">Build failed</a>"}'
|
||||
@ -0,0 +1,28 @@
|
||||
name: Validate migrations
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'priv/repo/migrations/**'
|
||||
- 'priv/ingest_repo/migrations/**'
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
name: App code does not change at the same time
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: changes
|
||||
with:
|
||||
list-files: json
|
||||
filters: |
|
||||
lib:
|
||||
- 'lib/**'
|
||||
- 'extra/**'
|
||||
- 'config/**'
|
||||
|
||||
- if: steps.changes.outputs.lib == 'true'
|
||||
run: |
|
||||
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
|
||||
exit 1
|
||||
@ -0,0 +1,5 @@
|
||||
Plausible copyright notice
|
||||
|
||||
This software is Copyright 2024 Plausible Insights OÜ
|
||||
|
||||
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io
|
||||
@ -0,0 +1,247 @@
|
||||
import React from 'react';
|
||||
import { withRouter } from 'react-router-dom'
|
||||
import Chart from 'chart.js/auto';
|
||||
import { navigateToQuery } from '../../query'
|
||||
import GraphTooltip from './graph-tooltip'
|
||||
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
|
||||
import dateFormatter from './date-formatter';
|
||||
import FadeIn from '../../fade-in';
|
||||
import classNames from 'classnames';
|
||||
|
||||
const calculateMaximumY = function(dataset) {
|
||||
const yAxisValues = dataset
|
||||
.flatMap((item) => item.data)
|
||||
.map((item) => item || 0)
|
||||
|
||||
if (yAxisValues) {
|
||||
return Math.max(...yAxisValues)
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
class LineGraph extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.regenerateChart = this.regenerateChart.bind(this);
|
||||
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
|
||||
}
|
||||
|
||||
getGraphMetric() {
|
||||
let metric = this.props.graphData.metric
|
||||
|
||||
if (metric == 'visitors' && this.props.query.filters.goal) {
|
||||
return 'conversions'
|
||||
} else {
|
||||
return metric
|
||||
}
|
||||
}
|
||||
|
||||
regenerateChart() {
|
||||
const { graphData, query } = this.props
|
||||
const metric = this.getGraphMetric()
|
||||
const graphEl = document.getElementById("main-graph-canvas")
|
||||
this.ctx = graphEl.getContext('2d');
|
||||
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
|
||||
|
||||
return new Chart(this.ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: graphData.labels,
|
||||
datasets: dataSet
|
||||
},
|
||||
options: {
|
||||
animation: false,
|
||||
plugins: {
|
||||
legend: { display: false },
|
||||
tooltip: {
|
||||
enabled: false,
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
position: 'average',
|
||||
external: GraphTooltip(graphData, metric, query)
|
||||
},
|
||||
},
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
onResize: this.updateWindowDimensions,
|
||||
elements: { line: { tension: 0 }, point: { radius: 0 } },
|
||||
onClick: this.maybeHopToHoveredPeriod.bind(this),
|
||||
scale: {
|
||||
ticks: { precision: 0, maxTicksLimit: 8 }
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
ticks: {
|
||||
callback: METRIC_FORMATTER[metric],
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
},
|
||||
grid: {
|
||||
zeroLineColor: 'transparent',
|
||||
drawBorder: false,
|
||||
}
|
||||
},
|
||||
yComparison: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
display: false,
|
||||
grid: { display: false },
|
||||
},
|
||||
x: {
|
||||
grid: { display: false },
|
||||
ticks: {
|
||||
callback: function(val, _index, _ticks) {
|
||||
if (this.getLabelForValue(val) == "__blank__") return ""
|
||||
|
||||
const hasMultipleYears =
|
||||
graphData.labels
|
||||
.filter((date) => typeof date === 'string')
|
||||
.map(date => date.split('-')[0])
|
||||
.filter((value, index, list) => list.indexOf(value) === index)
|
||||
.length > 1
|
||||
|
||||
if (graphData.interval === 'hour' && query.period !== 'day') {
|
||||
const date = dateFormatter({
|
||||
interval: "date",
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
const hour = dateFormatter({
|
||||
interval: graphData.interval,
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
// Returns a combination of date and hour. This is because
|
||||
// small intervals like hour may return multiple days
|
||||
// depending on the query period.
|
||||
return `${date}, ${hour}`
|
||||
}
|
||||
|
||||
if (graphData.interval === 'minute' && query.period !== 'realtime') {
|
||||
return dateFormatter({
|
||||
interval: "hour", longForm: false, period: query.period,
|
||||
})(this.getLabelForValue(val))
|
||||
}
|
||||
|
||||
return dateFormatter({
|
||||
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
},
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
}
|
||||
}
|
||||
},
|
||||
interaction: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
repositionTooltip(e) {
|
||||
const tooltipEl = document.getElementById('chartjs-tooltip');
|
||||
if (tooltipEl && window.innerWidth >= 768) {
|
||||
if (e.clientX > 0.66 * window.innerWidth) {
|
||||
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
|
||||
tooltipEl.style.left = null;
|
||||
} else {
|
||||
tooltipEl.style.right = null;
|
||||
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
|
||||
}
|
||||
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
|
||||
tooltipEl.style.opacity = 1;
|
||||
}
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
if (this.props.graphData) {
|
||||
this.chart = this.regenerateChart();
|
||||
}
|
||||
window.addEventListener('mousemove', this.repositionTooltip);
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps) {
|
||||
const { graphData, darkTheme } = this.props;
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
|
||||
if (
|
||||
graphData !== prevProps.graphData ||
|
||||
darkTheme !== prevProps.darkTheme
|
||||
) {
|
||||
|
||||
if (graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
this.chart = this.regenerateChart();
|
||||
this.chart.update();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (!graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
// Ensure that the tooltip doesn't hang around when we are loading more data
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
if (tooltip) {
|
||||
tooltip.style.opacity = 0;
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
window.removeEventListener('mousemove', this.repositionTooltip)
|
||||
}
|
||||
|
||||
/**
|
||||
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
|
||||
* @param {*} chart - The chart instance.
|
||||
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
|
||||
*/
|
||||
updateWindowDimensions(chart, dimensions) {
|
||||
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
|
||||
}
|
||||
|
||||
maybeHopToHoveredPeriod(e) {
|
||||
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
|
||||
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
|
||||
|
||||
if (this.props.graphData.interval === 'month') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
|
||||
} else if (this.props.graphData.interval === 'date') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
const { graphData } = this.props
|
||||
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
|
||||
|
||||
return (
|
||||
<FadeIn show={graphData}>
|
||||
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
|
||||
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
|
||||
</div>
|
||||
</FadeIn>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default withRouter(LineGraph)
|
||||
@ -0,0 +1,17 @@
|
||||
import React from "react"
|
||||
|
||||
export default function SamplingNotice({topStatData}) {
|
||||
const samplePercent = topStatData?.samplePercent
|
||||
|
||||
if (samplePercent && samplePercent < 100) {
|
||||
return (
|
||||
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
|
||||
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
import React, { useState } from "react"
|
||||
import * as api from '../../api'
|
||||
import { getCurrentInterval } from "./interval-picker"
|
||||
|
||||
export default function StatsExport({site, query}) {
|
||||
const [exporting, setExporting] = useState(false)
|
||||
|
||||
function startExport() {
|
||||
setExporting(true)
|
||||
document.cookie = "exporting="
|
||||
pollExportReady()
|
||||
}
|
||||
|
||||
function pollExportReady() {
|
||||
if (document.cookie.includes('exporting')) {
|
||||
setTimeout(pollExportReady, 1000)
|
||||
} else {
|
||||
setExporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
function renderLoading() {
|
||||
return (
|
||||
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function renderExportLink() {
|
||||
const interval = getCurrentInterval(site, query)
|
||||
const queryParams = api.serializeQuery(query, [{ interval }])
|
||||
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
|
||||
|
||||
return (
|
||||
<a href={endpoint} download onClick={startExport}>
|
||||
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
|
||||
<polyline points="7 10 12 15 17 10"></polyline>
|
||||
<line x1="12" y1="15" x2="12" y2="3"></line>
|
||||
</svg>
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-4 h-4 mx-2">
|
||||
{exporting && renderLoading()}
|
||||
{!exporting && renderExportLink()}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
import React from "react"
|
||||
import { parseNaiveDate, isBefore } from '../../util/date'
|
||||
import { Link } from 'react-router-dom'
|
||||
import * as url from '../../util/url'
|
||||
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
|
||||
|
||||
export default function WithImportedSwitch({site, topStatData}) {
|
||||
if (!topStatData?.imports_exist) {
|
||||
return null
|
||||
}
|
||||
|
||||
function isBeforeNativeStats(date) {
|
||||
if (!date) return false
|
||||
|
||||
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
|
||||
const parsedDate = parseNaiveDate(date)
|
||||
|
||||
return isBefore(parsedDate, nativeStatsBegin, "day")
|
||||
}
|
||||
|
||||
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
|
||||
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
|
||||
|
||||
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
|
||||
const withImported = topStatData.with_imported;
|
||||
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
|
||||
const target = url.setQuery('with_imported', !withImported)
|
||||
const tip = withImported ? "" : "do not ";
|
||||
|
||||
return (
|
||||
<Link to={target} className="w-4 h-4 mx-2">
|
||||
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
|
||||
<BarsArrowUpIcon className={"absolute " + toggleColor} />
|
||||
</div>
|
||||
</Link>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
Plausible copyright notice
|
||||
|
||||
This software is Copyright 2024 Plausible Insights OÜ
|
||||
|
||||
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io
|
||||
@ -0,0 +1,42 @@
|
||||
defmodule Plausible.License do
|
||||
@moduledoc """
|
||||
This module ensures that you cannot run Plausible Analytics Enterprise Edition without a valid license key.
|
||||
The software contained within the ee/ and assets/js/dashboard/ee directories are Copyright © Plausible Insights OÜ.
|
||||
We have made this code available solely for informational and transparency purposes. No rights are granted to use,
|
||||
distribute, or exploit this software in any form.
|
||||
|
||||
Any attempt to disable or modify the behavior of this module will be considered a violation of copyright.
|
||||
If you wish to use the Plausible Analytics Enterprise Edition for your own requirements, please contact us
|
||||
at hello@plausible.io to discuss obtaining a license.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
if Mix.env() == :prod do
|
||||
def ensure_valid_license do
|
||||
if has_valid_license?() do
|
||||
:ok
|
||||
else
|
||||
Logger.error(
|
||||
"Invalid or no license key provided for Plausible Enterprise Edition. Please contact hello@plausible.io to acquire a license."
|
||||
)
|
||||
|
||||
Logger.error("Shutting down")
|
||||
System.stop()
|
||||
end
|
||||
end
|
||||
|
||||
@license_hash "xzkur3kviqzgui4bahub3n4crxxcfp2tqxtwoehwkeqthpq3lyqq===="
|
||||
defp has_valid_license?() do
|
||||
hash =
|
||||
:crypto.hash(:sha256, Application.fetch_env!(:plausible, :license_key))
|
||||
|> Base.encode32(case: :lower)
|
||||
|
||||
hash == @license_hash
|
||||
end
|
||||
else
|
||||
def ensure_valid_license do
|
||||
:ok
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240302"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
{
|
||||
"account": "accounts/28425555",
|
||||
"createTime": "2024-02-22T10:50:15.462Z",
|
||||
"currencyCode": "USD",
|
||||
"displayName": "account.one - GA4",
|
||||
"name": "properties/428685444",
|
||||
"parent": "accounts/28425555",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY",
|
||||
"serviceLevel": "GOOGLE_ANALYTICS_STANDARD",
|
||||
"timeZone": "Europe/Warsaw",
|
||||
"updateTime": "2024-02-22T10:50:15.462Z"
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"accountSummaries": [
|
||||
{
|
||||
"account": "accounts/28425178",
|
||||
"displayName": "account.one",
|
||||
"name": "accountSummaries/28425178",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "account.one - GA4",
|
||||
"parent": "accounts/28425178",
|
||||
"property": "properties/428685906",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"account": "accounts/45336102",
|
||||
"displayName": "account.two",
|
||||
"name": "accountSummaries/45336102"
|
||||
},
|
||||
{
|
||||
"account": "accounts/54516992",
|
||||
"displayName": "Demo Account",
|
||||
"name": "accountSummaries/54516992",
|
||||
"propertySummaries": [
|
||||
{
|
||||
"displayName": "GA4 - Flood-It!",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/153293282",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
},
|
||||
{
|
||||
"displayName": "GA4 - Google Merch Shop",
|
||||
"parent": "accounts/54516992",
|
||||
"property": "properties/213025502",
|
||||
"propertyType": "PROPERTY_TYPE_ORDINARY"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,786 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Amsterdam"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "totalUsers",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "bounces",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "sessions",
|
||||
"type": "TYPE_INTEGER"
|
||||
},
|
||||
{
|
||||
"name": "userEngagementDuration",
|
||||
"type": "TYPE_SECONDS"
|
||||
}
|
||||
],
|
||||
"rowCount": 31,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240131"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "141"
|
||||
},
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "56"
|
||||
},
|
||||
{
|
||||
"value": "146"
|
||||
},
|
||||
{
|
||||
"value": "4815"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240130"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "42"
|
||||
},
|
||||
{
|
||||
"value": "115"
|
||||
},
|
||||
{
|
||||
"value": "3481"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240129"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "105"
|
||||
},
|
||||
{
|
||||
"value": "153"
|
||||
},
|
||||
{
|
||||
"value": "54"
|
||||
},
|
||||
{
|
||||
"value": "113"
|
||||
},
|
||||
{
|
||||
"value": "2895"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240128"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "184"
|
||||
},
|
||||
{
|
||||
"value": "259"
|
||||
},
|
||||
{
|
||||
"value": "74"
|
||||
},
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "6861"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240127"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "202"
|
||||
},
|
||||
{
|
||||
"value": "67"
|
||||
},
|
||||
{
|
||||
"value": "158"
|
||||
},
|
||||
{
|
||||
"value": "5259"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240126"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "132"
|
||||
},
|
||||
{
|
||||
"value": "189"
|
||||
},
|
||||
{
|
||||
"value": "57"
|
||||
},
|
||||
{
|
||||
"value": "142"
|
||||
},
|
||||
{
|
||||
"value": "5545"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240125"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "151"
|
||||
},
|
||||
{
|
||||
"value": "229"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "155"
|
||||
},
|
||||
{
|
||||
"value": "5365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240124"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "137"
|
||||
},
|
||||
{
|
||||
"value": "221"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "148"
|
||||
},
|
||||
{
|
||||
"value": "4454"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240123"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "166"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "171"
|
||||
},
|
||||
{
|
||||
"value": "4850"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240122"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "161"
|
||||
},
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "73"
|
||||
},
|
||||
{
|
||||
"value": "165"
|
||||
},
|
||||
{
|
||||
"value": "4871"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240121"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "254"
|
||||
},
|
||||
{
|
||||
"value": "397"
|
||||
},
|
||||
{
|
||||
"value": "114"
|
||||
},
|
||||
{
|
||||
"value": "267"
|
||||
},
|
||||
{
|
||||
"value": "8845"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240120"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "214"
|
||||
},
|
||||
{
|
||||
"value": "383"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "6914"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240119"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "381"
|
||||
},
|
||||
{
|
||||
"value": "5571"
|
||||
},
|
||||
{
|
||||
"value": "100"
|
||||
},
|
||||
{
|
||||
"value": "228"
|
||||
},
|
||||
{
|
||||
"value": "98549"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240118"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "177"
|
||||
},
|
||||
{
|
||||
"value": "278"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "186"
|
||||
},
|
||||
{
|
||||
"value": "5553"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240117"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "204"
|
||||
},
|
||||
{
|
||||
"value": "271"
|
||||
},
|
||||
{
|
||||
"value": "97"
|
||||
},
|
||||
{
|
||||
"value": "216"
|
||||
},
|
||||
{
|
||||
"value": "6777"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240116"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "200"
|
||||
},
|
||||
{
|
||||
"value": "280"
|
||||
},
|
||||
{
|
||||
"value": "65"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9381"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240115"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "255"
|
||||
},
|
||||
{
|
||||
"value": "311"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "12365"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240114"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "243"
|
||||
},
|
||||
{
|
||||
"value": "302"
|
||||
},
|
||||
{
|
||||
"value": "86"
|
||||
},
|
||||
{
|
||||
"value": "257"
|
||||
},
|
||||
{
|
||||
"value": "11073"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240113"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "231"
|
||||
},
|
||||
{
|
||||
"value": "276"
|
||||
},
|
||||
{
|
||||
"value": "92"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "10084"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240112"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "215"
|
||||
},
|
||||
{
|
||||
"value": "265"
|
||||
},
|
||||
{
|
||||
"value": "75"
|
||||
},
|
||||
{
|
||||
"value": "225"
|
||||
},
|
||||
{
|
||||
"value": "10195"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240111"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "183"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "64"
|
||||
},
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "8200"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240110"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "190"
|
||||
},
|
||||
{
|
||||
"value": "244"
|
||||
},
|
||||
{
|
||||
"value": "61"
|
||||
},
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "9056"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240109"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "201"
|
||||
},
|
||||
{
|
||||
"value": "290"
|
||||
},
|
||||
{
|
||||
"value": "63"
|
||||
},
|
||||
{
|
||||
"value": "213"
|
||||
},
|
||||
{
|
||||
"value": "11651"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240108"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "195"
|
||||
},
|
||||
{
|
||||
"value": "235"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "209"
|
||||
},
|
||||
{
|
||||
"value": "7946"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240107"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "205"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "80"
|
||||
},
|
||||
{
|
||||
"value": "218"
|
||||
},
|
||||
{
|
||||
"value": "10048"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240106"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "256"
|
||||
},
|
||||
{
|
||||
"value": "303"
|
||||
},
|
||||
{
|
||||
"value": "119"
|
||||
},
|
||||
{
|
||||
"value": "281"
|
||||
},
|
||||
{
|
||||
"value": "8440"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240105"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "226"
|
||||
},
|
||||
{
|
||||
"value": "270"
|
||||
},
|
||||
{
|
||||
"value": "83"
|
||||
},
|
||||
{
|
||||
"value": "237"
|
||||
},
|
||||
{
|
||||
"value": "9311"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240104"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "223"
|
||||
},
|
||||
{
|
||||
"value": "274"
|
||||
},
|
||||
{
|
||||
"value": "81"
|
||||
},
|
||||
{
|
||||
"value": "236"
|
||||
},
|
||||
{
|
||||
"value": "12051"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240103"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "233"
|
||||
},
|
||||
{
|
||||
"value": "283"
|
||||
},
|
||||
{
|
||||
"value": "79"
|
||||
},
|
||||
{
|
||||
"value": "245"
|
||||
},
|
||||
{
|
||||
"value": "10628"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240102"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "203"
|
||||
},
|
||||
{
|
||||
"value": "269"
|
||||
},
|
||||
{
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"value": "210"
|
||||
},
|
||||
{
|
||||
"value": "11168"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240101"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "191"
|
||||
},
|
||||
{
|
||||
"value": "224"
|
||||
},
|
||||
{
|
||||
"value": "71"
|
||||
},
|
||||
{
|
||||
"value": "197"
|
||||
},
|
||||
{
|
||||
"value": "8080"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240222"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20200421"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20120118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20170118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,136 @@
|
||||
defmodule Plausible.Cache.Adapter do
|
||||
@moduledoc """
|
||||
Interface for the underlying cache implementation.
|
||||
Currently: ConCache
|
||||
|
||||
Using the Adapter module directly, the user must ensure that the relevant
|
||||
processes are available to use, which is normally done via the child specification.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()
|
||||
def child_spec(name, child_id, opts \\ [])
|
||||
when is_atom(name) and is_atom(child_id) and is_list(opts) do
|
||||
cache_name = Keyword.get(opts, :cache_name, name)
|
||||
child_id = Keyword.get(opts, :child_id, child_id)
|
||||
ttl_check_interval = Keyword.get(opts, :ttl_check_interval, false)
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Keyword.put(:name, cache_name)
|
||||
|> Keyword.put(:ttl_check_interval, ttl_check_interval)
|
||||
|
||||
Supervisor.child_spec(
|
||||
{ConCache, opts},
|
||||
id: child_id
|
||||
)
|
||||
end
|
||||
|
||||
@spec size(atom()) :: non_neg_integer() | nil
|
||||
def size(cache_name) do
|
||||
ConCache.size(cache_name)
|
||||
catch
|
||||
:exit, _ -> nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any()) :: any()
|
||||
def get(cache_name, key) do
|
||||
result = ConCache.get(cache_name, key)
|
||||
if is_nil(result), do: miss(cache_name), else: hit(cache_name)
|
||||
result
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec get(atom(), any(), (-> any())) :: any()
|
||||
def get(cache_name, key, fallback_fn) do
|
||||
cache = ConCache.Owner.cache(cache_name)
|
||||
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
get_or_store_isolated(cache, cache_name, key, fallback_fn)
|
||||
|
||||
value ->
|
||||
hit(cache_name)
|
||||
value
|
||||
end
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put(atom(), any(), any()) :: any()
|
||||
def put(cache_name, key, value) do
|
||||
:ok = ConCache.put(cache_name, key, value)
|
||||
value
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting a key to '#{cache_name}'")
|
||||
nil
|
||||
end
|
||||
|
||||
@spec put_many(atom(), [any()]) :: :ok
|
||||
def put_many(cache_name, items) when is_list(items) do
|
||||
true = :ets.insert(ConCache.ets(cache_name), items)
|
||||
:ok
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error putting keys to '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec delete(atom(), any()) :: :ok
|
||||
def delete(cache_name, key) do
|
||||
ConCache.dirty_delete(cache_name, key)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error deleting a key in '#{cache_name}'")
|
||||
:ok
|
||||
end
|
||||
|
||||
@spec keys(atom()) :: Enumerable.t()
|
||||
def keys(cache_name) do
|
||||
ets = ConCache.ets(cache_name)
|
||||
|
||||
Stream.resource(
|
||||
fn -> :ets.first(ets) end,
|
||||
fn
|
||||
:"$end_of_table" -> {:halt, nil}
|
||||
prev_key -> {[prev_key], :ets.next(ets, prev_key)}
|
||||
end,
|
||||
fn _ -> :ok end
|
||||
)
|
||||
catch
|
||||
:exit, _ ->
|
||||
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
|
||||
[]
|
||||
end
|
||||
|
||||
defp hit(cache_name) do
|
||||
Plausible.Cache.Stats.record_hit(cache_name)
|
||||
end
|
||||
|
||||
defp miss(cache_name) do
|
||||
Plausible.Cache.Stats.record_miss(cache_name)
|
||||
end
|
||||
|
||||
defp get_or_store_isolated(cache, cache_name, key, fun) do
|
||||
ConCache.isolated(cache_name, key, fn ->
|
||||
case ConCache.Operations.get(cache, key) do
|
||||
nil ->
|
||||
new_value = fun.()
|
||||
ConCache.Operations.dirty_put(cache, key, new_value)
|
||||
miss(cache_name)
|
||||
new_value
|
||||
|
||||
existing ->
|
||||
hit(cache_name)
|
||||
existing
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,84 @@
|
||||
defmodule Plausible.Cache.Stats do
|
||||
@moduledoc """
|
||||
Keeps track of hit/miss ratio for various caches.
|
||||
"""
|
||||
|
||||
use GenServer
|
||||
|
||||
@hit :hit
|
||||
@miss :miss
|
||||
@telemetry_hit [:plausible, :cache, :adapter, @hit]
|
||||
@telemetry_miss [:plausible, :cache, :adapter, @miss]
|
||||
@telemetry_events [@telemetry_hit, @telemetry_miss]
|
||||
|
||||
def start_link(_opts) do
|
||||
GenServer.start_link(__MODULE__, nil)
|
||||
end
|
||||
|
||||
def record_hit(cache_name) do
|
||||
:telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def record_miss(cache_name) do
|
||||
:telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name})
|
||||
end
|
||||
|
||||
def init(nil) do
|
||||
__MODULE__ =
|
||||
:ets.new(__MODULE__, [
|
||||
:public,
|
||||
:named_table,
|
||||
:set,
|
||||
read_concurrency: true,
|
||||
write_concurrency: true
|
||||
])
|
||||
|
||||
:telemetry.attach_many(
|
||||
"plausible-cache-stats",
|
||||
@telemetry_events,
|
||||
&__MODULE__.handle_telemetry_event/4,
|
||||
nil
|
||||
)
|
||||
|
||||
{:ok, nil}
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @hit)
|
||||
end
|
||||
|
||||
def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do
|
||||
bump(cache_name, @miss)
|
||||
end
|
||||
|
||||
def gather(cache_name) do
|
||||
{:ok,
|
||||
%{
|
||||
hit_rate: hit_rate(cache_name),
|
||||
count: size(cache_name) || 0
|
||||
}}
|
||||
end
|
||||
|
||||
defdelegate size(cache_name), to: Plausible.Cache.Adapter
|
||||
|
||||
def bump(cache_name, type) do
|
||||
:ets.update_counter(
|
||||
__MODULE__,
|
||||
{cache_name, type},
|
||||
1,
|
||||
{{cache_name, type}, 0}
|
||||
)
|
||||
end
|
||||
|
||||
def hit_rate(cache_name) do
|
||||
hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)
|
||||
miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)
|
||||
hit_miss = hit + miss
|
||||
|
||||
if hit_miss == 0 do
|
||||
0.0
|
||||
else
|
||||
hit / hit_miss * 100
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,166 @@
|
||||
defmodule Plausible.DataMigration.PopulateEventSessionColumns do
|
||||
@moduledoc """
|
||||
Populates event session columns with data from sessions table.
|
||||
|
||||
Run via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.run"
|
||||
Kill via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.kill"
|
||||
Monitor via ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.report_progress"
|
||||
|
||||
Suggested to run in a screen/tmux session to be able to easily monitor
|
||||
|
||||
SQL files available at: priv/data_migrations/PopulateEventSessionColumns/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "PopulateEventSessionColumns", repo: Plausible.IngestRepo
|
||||
|
||||
require Logger
|
||||
|
||||
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#cache for meaning of these defaults
|
||||
@default_dictionary_config %{
|
||||
lifetime: 600_000,
|
||||
size_in_cells: 1_000_000,
|
||||
max_threads_for_updates: 6
|
||||
}
|
||||
|
||||
def run(opts \\ []) do
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("create-sessions-dictionary",
|
||||
cluster?: cluster?,
|
||||
dictionary_connection_params:
|
||||
Keyword.get(opts, :dictionary_connection_string, dictionary_connection_params()),
|
||||
dictionary_config: dictionary_config(opts)
|
||||
)
|
||||
|
||||
{partitions, _, _, _} = get_partitions(opts)
|
||||
|
||||
IO.puts("Starting mutation on #{length(partitions)} partition(s)")
|
||||
|
||||
for partition <- partitions do
|
||||
{:ok, _} =
|
||||
run_sql("update-table", [cluster?: cluster?, partition: partition],
|
||||
settings: [allow_nondeterministic_mutations: 1]
|
||||
)
|
||||
end
|
||||
|
||||
wait_until_mutations_complete(opts)
|
||||
|
||||
IO.puts("Mutations seem done, cleaning up!")
|
||||
{:ok, _} = run_sql("drop-sessions-dictionary", cluster?: cluster?)
|
||||
end
|
||||
|
||||
def kill(opts \\ []) do
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")
|
||||
|
||||
report_progress(opts)
|
||||
|
||||
IO.puts("Killing running mutations")
|
||||
{:ok, _} = run_sql("kill-running-mutations", cluster?: cluster?)
|
||||
end
|
||||
|
||||
def wait_until_mutations_complete(opts \\ []) do
|
||||
Process.sleep(5_000)
|
||||
in_progress? = report_progress(opts)
|
||||
|
||||
if in_progress? do
|
||||
wait_until_mutations_complete(opts)
|
||||
end
|
||||
end
|
||||
|
||||
def report_progress(opts \\ []) do
|
||||
{partitions, parts, min_partition, max_partition} = get_partitions(opts)
|
||||
|
||||
{:ok, %{rows: mutation_results}} =
|
||||
run_sql("get-mutations-progress",
|
||||
min_partition: min_partition,
|
||||
max_partition: max_partition
|
||||
)
|
||||
|
||||
[
|
||||
[
|
||||
mutations,
|
||||
parts_to_do,
|
||||
running_for,
|
||||
total_size,
|
||||
todo_size,
|
||||
progress,
|
||||
latest_fail_reason,
|
||||
_,
|
||||
_
|
||||
]
|
||||
] =
|
||||
mutation_results
|
||||
|
||||
{:ok, %{rows: [[merges]]}} = run_sql("get-merges-progress")
|
||||
{:ok, %{rows: disks}} = run_sql("get-disks")
|
||||
|
||||
IO.puts("\n\n#{Timex.now() |> Timex.format!("{ISO:Extended}")}")
|
||||
|
||||
# List partitions that need to run
|
||||
IO.puts(
|
||||
"Progress report for partitions #{Enum.min(partitions)}-#{Enum.max(partitions)} (parts: #{length(parts)})"
|
||||
)
|
||||
|
||||
IO.puts("Disks overview:")
|
||||
|
||||
for [name, path, full_space, total_space, full_percentage] <- disks do
|
||||
IO.puts(
|
||||
" #{name} at #{path} is at #{full_space}/#{total_space} (#{full_percentage}% full)"
|
||||
)
|
||||
end
|
||||
|
||||
IO.puts("Currently #{mutations} mutation(s) are running.")
|
||||
|
||||
if mutations > 0 do
|
||||
IO.puts(" To do #{parts_to_do} parts, #{todo_size}")
|
||||
IO.puts(" Out of #{length(parts)} parts, #{total_size}")
|
||||
IO.puts(" Running for #{format_duration(running_for)}")
|
||||
|
||||
if progress > 0 do
|
||||
estimated_time_left = round(running_for / progress / 100 - running_for)
|
||||
IO.puts(" Estimated #{progress}% done, #{format_duration(estimated_time_left)} left")
|
||||
end
|
||||
|
||||
if latest_fail_reason do
|
||||
IO.puts(" Some mutations might be failing. ClickHouse report: #{latest_fail_reason}")
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Currently #{merges} merge(s) are running relating to mutations.")
|
||||
|
||||
mutations > 0
|
||||
end
|
||||
|
||||
defp dictionary_config(opts) do
|
||||
@default_dictionary_config
|
||||
|> Map.merge(Keyword.get(opts, :dictionary_config, %{}))
|
||||
end
|
||||
|
||||
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context
|
||||
defp dictionary_connection_params() do
|
||||
Plausible.IngestRepo.config()
|
||||
|> Enum.map(fn
|
||||
{:database, database} -> "DB '#{database}'"
|
||||
{:username, username} -> "USER '#{username}'"
|
||||
{:password, password} -> "PASSWORD '#{password}'"
|
||||
_ -> nil
|
||||
end)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.join(" ")
|
||||
end
|
||||
|
||||
defp get_partitions(opts) do
|
||||
[min_partition, max_partition] = Keyword.get(opts, :partition_range, ["0", "999999"])
|
||||
|
||||
{:ok, %{rows: [[partitions, parts]]}} =
|
||||
run_sql("list-partitions", min_partition: min_partition, max_partition: max_partition)
|
||||
|
||||
{partitions, parts, min_partition, max_partition}
|
||||
end
|
||||
|
||||
defp format_duration(seconds) do
|
||||
seconds
|
||||
|> Timex.Duration.from_seconds()
|
||||
|> Timex.format_duration(Timex.Format.Duration.Formatters.Humanized)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,188 @@
|
||||
defmodule Plausible.DataMigration.SiteImports do
|
||||
@moduledoc """
|
||||
Site imports migration backfilling SiteImport entries for old imports
|
||||
and alters import end dates to match actual end date of respective import stats.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.ClickhouseRepo
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
def run(opts \\ []) do
|
||||
dry_run? = Keyword.get(opts, :dry_run?, true)
|
||||
|
||||
site_import_query =
|
||||
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
|
||||
|
||||
sites_with_imports =
|
||||
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|
||||
|> Repo.all(log: false)
|
||||
|
||||
sites_count = length(sites_with_imports)
|
||||
|
||||
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
|
||||
|
||||
for {site, idx} <- Enum.with_index(sites_with_imports) do
|
||||
site_imports =
|
||||
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all(log: false)
|
||||
|
||||
IO.puts(
|
||||
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
|
||||
)
|
||||
|
||||
site_imports =
|
||||
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
|
||||
IO.puts("Creating legacy site import entry for site ID #{site.id}")
|
||||
|
||||
# create legacy entry if there's not one yet
|
||||
params =
|
||||
site.imported_data
|
||||
|> Imported.SiteImport.from_legacy()
|
||||
|> Map.put(:site_id, site.id)
|
||||
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
|
||||
|
||||
legacy_site_import =
|
||||
%Imported.SiteImport{}
|
||||
|> Ecto.Changeset.change(params)
|
||||
|> insert!(dry_run?)
|
||||
|
||||
[legacy_site_import | site_imports]
|
||||
else
|
||||
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
|
||||
|
||||
site_imports
|
||||
end
|
||||
|
||||
# adjust end date for each site import
|
||||
for site_import <- site_imports do
|
||||
IO.puts(
|
||||
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
|
||||
)
|
||||
|
||||
import_ids =
|
||||
if site_import.legacy do
|
||||
[0, site_import.id]
|
||||
else
|
||||
[site_import.id]
|
||||
end
|
||||
|
||||
end_date = imported_stats_end_date(site.id, import_ids)
|
||||
|
||||
if !end_date do
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
|
||||
)
|
||||
|
||||
if site_import.legacy do
|
||||
clear_imported_data(site, dry_run?)
|
||||
end
|
||||
|
||||
delete!(site_import, dry_run?)
|
||||
else
|
||||
end_date =
|
||||
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
|
||||
end_date
|
||||
else
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
|
||||
)
|
||||
|
||||
site_import.end_date
|
||||
end
|
||||
|
||||
site_import
|
||||
|> Ecto.Changeset.change(end_date: end_date)
|
||||
|> update!(dry_run?)
|
||||
|
||||
IO.puts(
|
||||
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
IO.puts("Done processing site ID #{site.id}")
|
||||
end
|
||||
|
||||
IO.puts("Finished")
|
||||
end
|
||||
|
||||
# Exposed for testing purposes
|
||||
@doc false
|
||||
def imported_stats_end_date(site_id, import_ids) do
|
||||
[first_schema | schemas] = Imported.schemas()
|
||||
|
||||
query =
|
||||
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
|
||||
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
|
||||
end)
|
||||
|
||||
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
|
||||
|
||||
if dates != [] do
|
||||
case Enum.max(dates, Date) do
|
||||
# no stats for this domain yet
|
||||
~D[1970-01-01] ->
|
||||
nil
|
||||
|
||||
date ->
|
||||
date
|
||||
end
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp insert!(changeset, false = _dry_run?) do
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
defp insert!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
changeset
|
||||
|> Ecto.Changeset.change(id: 0)
|
||||
|> Ecto.Changeset.apply_changes()
|
||||
else
|
||||
raise "Invalid insert: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, false = _dry_run?) do
|
||||
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
|
||||
end
|
||||
|
||||
defp clear_imported_data(site, true = _dry_run?) do
|
||||
%{site | imported_data: nil}
|
||||
end
|
||||
|
||||
defp update!(changeset, false = _dry_run?) do
|
||||
Repo.update!(changeset)
|
||||
end
|
||||
|
||||
defp update!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
Ecto.Changeset.apply_changes(changeset)
|
||||
else
|
||||
raise "Invalid update: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp delete!(entity, false = _dry_run?) do
|
||||
Repo.delete!(entity)
|
||||
end
|
||||
|
||||
defp delete!(entity, true = _dry_run?) do
|
||||
entity
|
||||
end
|
||||
|
||||
defp max_date_query(schema, site_id, import_ids) do
|
||||
from(q in schema,
|
||||
where: q.site_id == ^site_id,
|
||||
where: q.import_id in ^import_ids,
|
||||
select: %{max_date: fragment("max(?)", q.date)}
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,75 @@
|
||||
defmodule Plausible.DataMigration.VersionedSessions do
|
||||
@moduledoc """
|
||||
Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
|
||||
priv/data_migrations/VersionedSessions/sql
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
|
||||
|
||||
@suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
|
||||
@versioned_table_engines [
|
||||
"ReplicatedVersionedCollapsingMergeTree",
|
||||
"VersionedCollapsingMergeTree"
|
||||
]
|
||||
|
||||
def run(opts \\ []) do
|
||||
run_exchange? = Keyword.get(opts, :run_exchange?, true)
|
||||
|
||||
unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
|
||||
|
||||
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
|
||||
|
||||
{:ok, %{rows: partitions}} = run_sql("list-partitions")
|
||||
partitions = Enum.map(partitions, fn [part] -> part end)
|
||||
|
||||
{:ok, %{rows: [[current_table_engine, table_settings]]}} =
|
||||
run_sql("get-sessions-table-settings")
|
||||
|
||||
if Enum.member?(@versioned_table_engines, current_table_engine) do
|
||||
IO.puts("sessions_v2 table is already versioned, no migration needed")
|
||||
else
|
||||
{:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("create-sessions-tmp-table",
|
||||
cluster?: cluster?,
|
||||
table_settings: table_settings,
|
||||
unique_suffix: unique_suffix
|
||||
)
|
||||
|
||||
for partition <- partitions do
|
||||
{:ok, _} = run_sql("attach-partition", partition: partition)
|
||||
end
|
||||
|
||||
if run_exchange? do
|
||||
run_exchange(cluster?)
|
||||
end
|
||||
|
||||
IO.puts("Migration done!")
|
||||
end
|
||||
end
|
||||
|
||||
defp run_exchange(cluster?) do
|
||||
case run_sql("exchange-sessions-tables", cluster?: cluster?) do
|
||||
{:ok, _} ->
|
||||
nil
|
||||
|
||||
# Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
|
||||
{:error, %Ch.Error{code: 1}} ->
|
||||
IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2",
|
||||
to: "sessions_v2_backup",
|
||||
cluster?: cluster?
|
||||
)
|
||||
|
||||
{:ok, _} =
|
||||
run_sql("rename-table",
|
||||
from: "sessions_v2_tmp_versioned",
|
||||
to: "sessions_v2",
|
||||
cluster?: cluster?
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,371 @@
|
||||
defmodule Plausible.Exports do
|
||||
@moduledoc """
|
||||
Contains functions to export data for events and sessions as Zip archives.
|
||||
"""
|
||||
|
||||
require Plausible
|
||||
import Ecto.Query
|
||||
|
||||
@doc """
|
||||
Renders filename for the Zip archive containing the exported CSV files.
|
||||
|
||||
Examples:
|
||||
|
||||
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"plausible_io_20210101_20241231.zip"
|
||||
|
||||
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
|
||||
"Bücher_example_20210101_20241231.zip"
|
||||
|
||||
"""
|
||||
def archive_filename(domain, min_date, max_date) do
|
||||
name =
|
||||
Enum.join(
|
||||
[
|
||||
String.replace(domain, ".", "_"),
|
||||
Calendar.strftime(min_date, "%Y%m%d"),
|
||||
Calendar.strftime(max_date, "%Y%m%d")
|
||||
],
|
||||
"_"
|
||||
)
|
||||
|
||||
name <> ".zip"
|
||||
end
|
||||
|
||||
@doc """
|
||||
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
|
||||
tables into the format of `imported_*` tables for a website.
|
||||
"""
|
||||
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
|
||||
%{String.t() => Ecto.Query.t()}
|
||||
def export_queries(site_id, opts \\ []) do
|
||||
extname = opts[:extname] || ".csv"
|
||||
date_range = opts[:date_range]
|
||||
|
||||
filename = fn table ->
|
||||
name =
|
||||
if date_range do
|
||||
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
|
||||
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
|
||||
"#{table}_#{first_date}_#{last_date}"
|
||||
else
|
||||
table
|
||||
end
|
||||
|
||||
name <> extname
|
||||
end
|
||||
|
||||
%{
|
||||
filename.("imported_visitors") => export_visitors_q(site_id),
|
||||
filename.("imported_sources") => export_sources_q(site_id),
|
||||
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
|
||||
filename.("imported_pages") => export_pages_q(site_id),
|
||||
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
|
||||
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
|
||||
filename.("imported_locations") => export_locations_q(site_id),
|
||||
filename.("imported_devices") => export_devices_q(site_id),
|
||||
filename.("imported_browsers") => export_browsers_q(site_id),
|
||||
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
|
||||
}
|
||||
end
|
||||
|
||||
Plausible.on_full_build do
|
||||
defp sampled(table) do
|
||||
Plausible.Stats.Sampling.add_query_hint(from(table))
|
||||
end
|
||||
else
|
||||
defp sampled(table) do
|
||||
table
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop date(timestamp) do
|
||||
quote do
|
||||
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visit_duration(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
|
||||
:visit_duration
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visitors(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
|
||||
:visitors
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visits(t) do
|
||||
quote do
|
||||
selected_as(sum(unquote(t).sign), :visits)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop bounces(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
|
||||
:bounces
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop pageviews(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
|
||||
:pageviews
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_visitors_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
visitors(s),
|
||||
pageviews(s),
|
||||
bounces(s),
|
||||
visits(s),
|
||||
visit_duration(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_sources_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [
|
||||
selected_as(:date),
|
||||
selected_as(:source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term
|
||||
],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.referrer_source, :source),
|
||||
s.referrer,
|
||||
s.utm_source,
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term,
|
||||
pageviews(s),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_pages_q(site_id) do
|
||||
window_q =
|
||||
from e in sampled("events_v2"),
|
||||
where: e.site_id == ^site_id,
|
||||
select: %{
|
||||
timestamp: e.timestamp,
|
||||
next_timestamp:
|
||||
over(fragment("leadInFrame(?)", e.timestamp),
|
||||
partition_by: e.session_id,
|
||||
order_by: e.timestamp,
|
||||
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
|
||||
),
|
||||
pathname: e.pathname,
|
||||
hostname: e.hostname,
|
||||
name: e.name,
|
||||
user_id: e.user_id,
|
||||
session_id: e.session_id,
|
||||
_sample_factor: fragment("_sample_factor")
|
||||
}
|
||||
|
||||
from e in subquery(window_q),
|
||||
group_by: [selected_as(:date), e.pathname],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(e.timestamp),
|
||||
selected_as(fragment("any(?)", e.hostname), :hostname),
|
||||
selected_as(e.pathname, :page),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
|
||||
:visits
|
||||
),
|
||||
visitors(e),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
),
|
||||
# NOTE: are exits pageviews or any events?
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
|
||||
:exits
|
||||
),
|
||||
selected_as(
|
||||
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
|
||||
:time_on_page
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_entry_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.entry_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.entry_page,
|
||||
visitors(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:entrances
|
||||
),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_exit_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.exit_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.exit_page,
|
||||
visitors(s),
|
||||
visit_duration(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:exits
|
||||
),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_locations_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
|
||||
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.country_code, :country),
|
||||
selected_as(s.subdivision1_code, :region),
|
||||
selected_as(s.city_geoname_id, :city),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_devices_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.screen_size],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.screen_size, :device),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_browsers_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.browser, s.browser_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.browser,
|
||||
s.browser_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_operating_systems_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.operating_system,
|
||||
s.operating_system_version,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s),
|
||||
pageviews(s)
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Creates a streamable Zip archive from the provided (named) Ecto queries.
|
||||
|
||||
Example usage:
|
||||
|
||||
{:ok, pool} = Ch.start_link(pool_size: 1)
|
||||
|
||||
DBConnection.run(pool, fn conn ->
|
||||
conn
|
||||
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!("export.zip"))
|
||||
|> Stream.run()
|
||||
end)
|
||||
|
||||
"""
|
||||
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
|
||||
Enumerable.t()
|
||||
def stream_archive(conn, named_queries, opts \\ []) do
|
||||
entries =
|
||||
Enum.map(named_queries, fn {name, query} ->
|
||||
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
|
||||
|
||||
datastream =
|
||||
conn
|
||||
|> Ch.stream(sql, params, opts)
|
||||
|> Stream.map(fn %Ch.Result{data: data} -> data end)
|
||||
|
||||
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
|
||||
end)
|
||||
|
||||
Zstream.zip(entries)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,152 @@
|
||||
defmodule Plausible.Google.GA4.API do
|
||||
@moduledoc """
|
||||
API for Google Analytics 4.
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.GA4
|
||||
|
||||
require Logger
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 50_000
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
def list_properties(access_token) do
|
||||
case GA4.HTTP.list_accounts_for_user(access_token) do
|
||||
{:ok, %{"accountSummaries" => accounts}} ->
|
||||
accounts =
|
||||
accounts
|
||||
|> Enum.filter(& &1["propertySummaries"])
|
||||
|> Enum.map(fn account ->
|
||||
%{"account" => account_id, "displayName" => account_name} = account
|
||||
|
||||
{"#{account_name} (#{account_id})",
|
||||
Enum.map(account["propertySummaries"], fn property ->
|
||||
%{"displayName" => property_name, "property" => property_id} = property
|
||||
|
||||
{"#{property_name} (#{property_id})", property_id}
|
||||
end)}
|
||||
end)
|
||||
|
||||
{:ok, accounts}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, lookup_property) do
|
||||
case GA4.HTTP.get_property(access_token, lookup_property) do
|
||||
{:ok, property} ->
|
||||
%{"displayName" => property_name, "name" => property_id, "account" => account_id} =
|
||||
property
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
id: property_id,
|
||||
name: "#{property_name} (#{property_id})",
|
||||
account_id: account_id
|
||||
}}
|
||||
|
||||
{:error, cause} ->
|
||||
{:error, cause}
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_start_date(access_token, property)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
GA4.HTTP.get_analytics_end_date(access_token, property)
|
||||
end
|
||||
|
||||
def import_analytics(date_range, property, auth, persist_fn) do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
|
||||
)
|
||||
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, property, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, property, access_token, persist_fn) do
|
||||
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
|
||||
)
|
||||
|
||||
report_request = prepare_request(report_request, date_range, property, access_token)
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case GA4.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, row_count}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
|
||||
)
|
||||
|
||||
if report_request.offset + @per_page < row_count do
|
||||
fetch_and_persist(
|
||||
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
|
||||
)
|
||||
|
||||
{:error, cause}
|
||||
else
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
|
||||
)
|
||||
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp prepare_request(report_request, date_range, property, access_token) do
|
||||
%GA4.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
property: property,
|
||||
access_token: access_token,
|
||||
offset: 0,
|
||||
limit: @per_page
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,244 @@
|
||||
defmodule Plausible.Google.GA4.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Google Analytics 4 API.
|
||||
"""
|
||||
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
require Logger
|
||||
|
||||
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
|
||||
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: report_request.property,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
|
||||
metrics: Enum.map(report_request.metrics, &build_metric/1),
|
||||
orderBys: [
|
||||
%{
|
||||
dimension: %{
|
||||
dimensionName: "date",
|
||||
orderType: "ALPHANUMERIC"
|
||||
},
|
||||
desc: true
|
||||
}
|
||||
],
|
||||
limit: report_request.limit,
|
||||
offset: report_request.offset
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url =
|
||||
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
url,
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
row_count <- Map.fetch!(report, "rowCount"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, row_count}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.debug(
|
||||
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
|
||||
)
|
||||
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp build_metric(expression) do
|
||||
case String.split(expression, " = ") do
|
||||
[name, expression] ->
|
||||
%{
|
||||
name: name,
|
||||
expression: expression
|
||||
}
|
||||
|
||||
[name] ->
|
||||
%{name: name}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"rows" => rows,
|
||||
"dimensionHeaders" => dimension_headers,
|
||||
"metricHeaders" => metric_headers
|
||||
})
|
||||
when is_list(rows) do
|
||||
dimension_headers = Enum.map(dimension_headers, & &1["name"])
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
|
||||
dimension_values = Enum.map(dimensions, & &1["value"])
|
||||
metric_values = Enum.map(metrics, & &1["value"])
|
||||
metrics = Enum.zip(metric_headers, metric_values)
|
||||
dimensions = Enum.zip(dimension_headers, dimension_values)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_accounts_for_user(access_token) do
|
||||
url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
def get_property(access_token, property) do
|
||||
url = "#{admin_api_url()}/v1beta/#{property}"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
|
||||
{:error, :not_found}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 property #{property}",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2015-08-14"
|
||||
|
||||
def get_analytics_start_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, property) do
|
||||
get_analytics_boundary_date(access_token, property, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, property, edge) do
|
||||
descending? = edge == :end
|
||||
|
||||
params = %{
|
||||
requests: [
|
||||
%{
|
||||
property: "#{property}",
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "date"}],
|
||||
metrics: [%{name: "screenPageViews"}],
|
||||
orderBys: [
|
||||
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
|
||||
],
|
||||
limit: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["rows"] do
|
||||
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving GA4 #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
|
||||
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
|
||||
end
|
||||
@ -0,0 +1,135 @@
|
||||
defmodule Plausible.Google.GA4.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Google Analytics 4 API
|
||||
"""
|
||||
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
:metrics,
|
||||
:date_range,
|
||||
:property,
|
||||
:access_token,
|
||||
:offset,
|
||||
:limit
|
||||
]
|
||||
|
||||
@type t() :: %__MODULE__{
|
||||
dataset: String.t(),
|
||||
dimensions: [String.t()],
|
||||
metrics: [String.t()],
|
||||
date_range: Date.Range.t(),
|
||||
property: term(),
|
||||
access_token: String.t(),
|
||||
offset: non_neg_integer(),
|
||||
limit: non_neg_integer()
|
||||
}
|
||||
|
||||
def full_report do
|
||||
[
|
||||
%__MODULE__{
|
||||
dataset: "imported_visitors",
|
||||
dimensions: ["date"],
|
||||
metrics: [
|
||||
"totalUsers",
|
||||
"screenPageViews",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"sessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_sources",
|
||||
dimensions: [
|
||||
"date",
|
||||
"sessionSource",
|
||||
"sessionMedium",
|
||||
"sessionCampaignName",
|
||||
"sessionManualAdContent",
|
||||
"sessionGoogleAdsKeyword",
|
||||
"pageReferrer"
|
||||
],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_pages",
|
||||
dimensions: ["date", "hostName", "pagePath"],
|
||||
# NOTE: no exits as GA4 DATA API does not provide that metric
|
||||
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_entry_pages",
|
||||
dimensions: ["date", "landingPage"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"userEngagementDuration",
|
||||
"bounces = sessions - engagedSessions"
|
||||
]
|
||||
},
|
||||
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
|
||||
# %__MODULE__{
|
||||
# dataset: "imported_exit_pages",
|
||||
# dimensions: ["date", "ga:exitPagePath"],
|
||||
# metrics: [
|
||||
# "totalUsers",
|
||||
# "sessions",
|
||||
# "screenPageViews",
|
||||
# "userEngagementDuration",
|
||||
# "bounces = sessions - engagedSessions"
|
||||
# ]
|
||||
# },
|
||||
%__MODULE__{
|
||||
dataset: "imported_locations",
|
||||
dimensions: ["date", "countryId", "region", "city"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_devices",
|
||||
dimensions: ["date", "deviceCategory"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_browsers",
|
||||
dimensions: ["date", "browser"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_operating_systems",
|
||||
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
|
||||
metrics: [
|
||||
"screenPageViews",
|
||||
"totalUsers",
|
||||
"sessions",
|
||||
"bounces = sessions - engagedSessions",
|
||||
"userEngagementDuration"
|
||||
]
|
||||
}
|
||||
]
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,155 @@
|
||||
defmodule Plausible.Google.UA.API do
|
||||
@moduledoc """
|
||||
API for Universal Analytics
|
||||
"""
|
||||
|
||||
alias Plausible.Google
|
||||
alias Plausible.Google.UA
|
||||
|
||||
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
|
||||
|
||||
@type import_auth :: {
|
||||
access_token :: String.t(),
|
||||
refresh_token :: String.t(),
|
||||
expires_at :: String.t()
|
||||
}
|
||||
|
||||
@per_page 7_500
|
||||
@backoff_factor :timer.seconds(10)
|
||||
@max_attempts 5
|
||||
|
||||
@spec list_views(access_token :: String.t()) ::
|
||||
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
|
||||
@doc """
|
||||
Lists Google Analytics views grouped by hostname.
|
||||
"""
|
||||
def list_views(access_token) do
|
||||
case UA.HTTP.list_views_for_user(access_token) do
|
||||
{:ok, %{"items" => views}} ->
|
||||
views =
|
||||
views
|
||||
|> Enum.group_by(&view_hostname/1, &view_names/1)
|
||||
|> Enum.sort_by(fn {key, _} -> key end)
|
||||
|
||||
{:ok, views}
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
|
||||
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
|
||||
@doc """
|
||||
Returns a single Google Analytics view if the user has access to it.
|
||||
"""
|
||||
def get_view(access_token, lookup_id) do
|
||||
with {:ok, views} <- list_views(access_token) do
|
||||
views =
|
||||
views
|
||||
|> Enum.map(&elem(&1, 1))
|
||||
|> List.flatten()
|
||||
|
||||
case Enum.find(views, fn {_name, id} -> id == lookup_id end) do
|
||||
{view_name, view_id} ->
|
||||
{:ok, %{id: view_id, name: "#{view_name}"}}
|
||||
|
||||
nil ->
|
||||
{:error, :not_found}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_start_date(access_token, view_id)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
UA.HTTP.get_analytics_end_date(access_token, view_id)
|
||||
end
|
||||
|
||||
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
|
||||
:ok | {:error, term()}
|
||||
@doc """
|
||||
Imports stats from a Google Analytics UA view to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics reports in batches of #{@per_page} per
|
||||
request. The batches are then passed to persist callback.
|
||||
|
||||
Requests to Google Analytics can fail, and are retried at most
|
||||
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
|
||||
importing has finished or `{:error, term()}` when a request to GA failed too
|
||||
many times.
|
||||
|
||||
Useful links:
|
||||
|
||||
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
|
||||
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
|
||||
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
|
||||
|
||||
"""
|
||||
def import_analytics(date_range, view_id, auth, persist_fn) do
|
||||
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
|
||||
do_import_analytics(date_range, view_id, access_token, persist_fn)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
|
||||
Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
|
||||
report_request = %UA.ReportRequest{
|
||||
report_request
|
||||
| date_range: date_range,
|
||||
view_id: view_id,
|
||||
access_token: access_token,
|
||||
page_token: nil,
|
||||
page_size: @per_page
|
||||
}
|
||||
|
||||
case fetch_and_persist(report_request, persist_fn: persist_fn) do
|
||||
:ok -> {:cont, :ok}
|
||||
{:error, _} = error -> {:halt, error}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
|
||||
:ok | {:error, term()}
|
||||
def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
|
||||
persist_fn = Keyword.fetch!(opts, :persist_fn)
|
||||
attempt = Keyword.get(opts, :attempt, 1)
|
||||
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
|
||||
|
||||
case UA.HTTP.get_report(report_request) do
|
||||
{:ok, {rows, next_page_token}} ->
|
||||
:ok = persist_fn.(report_request.dataset, rows)
|
||||
|
||||
if next_page_token do
|
||||
fetch_and_persist(
|
||||
%UA.ReportRequest{report_request | page_token: next_page_token},
|
||||
opts
|
||||
)
|
||||
else
|
||||
:ok
|
||||
end
|
||||
|
||||
{:error, cause} ->
|
||||
if attempt >= @max_attempts do
|
||||
{:error, cause}
|
||||
else
|
||||
Process.sleep(attempt * sleep_time)
|
||||
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp view_hostname(view) do
|
||||
case view do
|
||||
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
|
||||
_any -> "Others"
|
||||
end
|
||||
end
|
||||
|
||||
defp view_names(%{"name" => name, "id" => id}) do
|
||||
{"#{id} - #{name}", id}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,191 @@
|
||||
defmodule Plausible.Google.UA.HTTP do
|
||||
@moduledoc """
|
||||
HTTP client implementation for Universal Analytics API.
|
||||
"""
|
||||
|
||||
require Logger
|
||||
alias Plausible.HTTPClient
|
||||
|
||||
@spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
|
||||
{:ok, {[map()], String.t() | nil}} | {:error, any()}
|
||||
def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: report_request.view_id,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
|
||||
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
|
||||
pageSize: report_request.page_size,
|
||||
pageToken: report_request.page_token
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
response =
|
||||
HTTPClient.impl().post(
|
||||
"#{reporting_api_url()}/v4/reports:batchGet",
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params,
|
||||
receive_timeout: 60_000
|
||||
)
|
||||
|
||||
with {:ok, %{body: body}} <- response,
|
||||
{:ok, report} <- parse_report_from_response(body),
|
||||
token <- Map.get(report, "nextPageToken"),
|
||||
{:ok, report} <- convert_to_maps(report) do
|
||||
{:ok, {report, token}}
|
||||
else
|
||||
{:error, %{reason: %{status: status, body: body}}} ->
|
||||
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
|
||||
{:error, :request_failed}
|
||||
|
||||
{:error, _reason} ->
|
||||
{:error, :request_failed}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_report_from_response(%{"reports" => [report | _]}) do
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp parse_report_from_response(body) do
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: body})
|
||||
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
|
||||
)
|
||||
|
||||
{:error, {:invalid_response, body}}
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"data" => %{} = data,
|
||||
"columnHeader" => %{
|
||||
"dimensions" => dimension_headers,
|
||||
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
|
||||
}
|
||||
}) do
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
rows = Map.get(data, "rows", [])
|
||||
|
||||
report =
|
||||
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
|
||||
metrics = Enum.zip(metric_headers, metrics)
|
||||
dimensions = Enum.zip(dimension_headers, dimensions)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
|
||||
{:ok, report}
|
||||
end
|
||||
|
||||
defp convert_to_maps(response) do
|
||||
Logger.error(
|
||||
"Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
|
||||
)
|
||||
|
||||
Sentry.Context.set_extra_context(%{universal_analytics_response: response})
|
||||
{:error, {:invalid_response, response}}
|
||||
end
|
||||
|
||||
def list_views_for_user(access_token) do
|
||||
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
|
||||
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().get(url, headers) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
{:ok, body}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2005-01-01"
|
||||
|
||||
def get_analytics_start_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :start)
|
||||
end
|
||||
|
||||
def get_analytics_end_date(access_token, view_id) do
|
||||
get_analytics_boundary_date(access_token, view_id, :end)
|
||||
end
|
||||
|
||||
defp get_analytics_boundary_date(access_token, view_id, edge) do
|
||||
sort_order =
|
||||
if edge == :start do
|
||||
"ASCENDING"
|
||||
else
|
||||
"DESCENDING"
|
||||
end
|
||||
|
||||
params = %{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: view_id,
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "ga:date", histogramBuckets: []}],
|
||||
metrics: [%{expression: "ga:pageviews"}],
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}],
|
||||
pageSize: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
url = "#{reporting_api_url()}/v4/reports:batchGet"
|
||||
headers = [{"Authorization", "Bearer #{access_token}"}]
|
||||
|
||||
case HTTPClient.impl().post(url, headers, params) do
|
||||
{:ok, %Finch.Response{body: body, status: 200}} ->
|
||||
report = List.first(body["reports"])
|
||||
|
||||
date =
|
||||
case report["data"]["rows"] do
|
||||
[%{"dimensions" => [date_str]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
|
||||
{:error, :authentication_failed}
|
||||
|
||||
{:error, %{reason: :timeout}} ->
|
||||
{:error, :timeout}
|
||||
|
||||
{:error, error} ->
|
||||
Sentry.capture_message("Error retrieving UA #{edge} date",
|
||||
extra: %{error: error}
|
||||
)
|
||||
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp config, do: Application.get_env(:plausible, :google)
|
||||
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
|
||||
defp api_url, do: Keyword.fetch!(config(), :api_url)
|
||||
end
|
||||
@ -1,4 +1,8 @@
|
||||
defmodule Plausible.Google.ReportRequest do
|
||||
defmodule Plausible.Google.UA.ReportRequest do
|
||||
@moduledoc """
|
||||
Report request struct for Universal Analytics API
|
||||
"""
|
||||
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
@ -0,0 +1,276 @@
|
||||
defmodule Plausible.Imported.GoogleAnalytics4 do
|
||||
@moduledoc """
|
||||
Import implementation for Google Analytics 4.
|
||||
"""
|
||||
|
||||
use Plausible.Imported.Importer
|
||||
|
||||
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
|
||||
|
||||
@impl true
|
||||
def name(), do: :google_analytics_4
|
||||
|
||||
@impl true
|
||||
def label(), do: "Google Analytics 4"
|
||||
|
||||
@impl true
|
||||
def email_template(), do: "google_analytics_import.html"
|
||||
|
||||
@impl true
|
||||
def parse_args(
|
||||
%{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
|
||||
) do
|
||||
start_date = Date.from_iso8601!(start_date)
|
||||
end_date = Date.from_iso8601!(end_date)
|
||||
date_range = Date.range(start_date, end_date)
|
||||
|
||||
auth = {
|
||||
Map.fetch!(args, "access_token"),
|
||||
Map.fetch!(args, "refresh_token"),
|
||||
Map.fetch!(args, "token_expires_at")
|
||||
}
|
||||
|
||||
[
|
||||
property: property,
|
||||
date_range: date_range,
|
||||
auth: auth
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Imports stats from a Google Analytics 4 property to a Plausible site.
|
||||
|
||||
This function fetches Google Analytics 4 reports which are then passed in batches
|
||||
to Clickhouse by the `Plausible.Imported.Buffer` process.
|
||||
"""
|
||||
@impl true
|
||||
def import_data(site_import, opts) do
|
||||
date_range = Keyword.fetch!(opts, :date_range)
|
||||
property = Keyword.fetch!(opts, :property)
|
||||
auth = Keyword.fetch!(opts, :auth)
|
||||
flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
|
||||
|
||||
{:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
|
||||
|
||||
persist_fn = fn table, rows ->
|
||||
records = from_report(rows, site_import.site_id, site_import.id, table)
|
||||
Plausible.Imported.Buffer.insert_many(buffer, table, records)
|
||||
end
|
||||
|
||||
try do
|
||||
Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
|
||||
after
|
||||
Plausible.Imported.Buffer.flush(buffer)
|
||||
Plausible.Imported.Buffer.stop(buffer)
|
||||
end
|
||||
end
|
||||
|
||||
def from_report(nil, _site_id, _import_id, _metric), do: nil
|
||||
|
||||
def from_report(data, site_id, import_id, table) do
|
||||
Enum.reduce(data, [], fn row, acc ->
|
||||
if Map.get(row.dimensions, "date") in @missing_values do
|
||||
acc
|
||||
else
|
||||
[new_from_report(site_id, import_id, table, row) | acc]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp parse_number(nr) do
|
||||
{float, ""} = Float.parse(nr)
|
||||
round(float)
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_visitors", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_sources", row) do
|
||||
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
|
||||
|
||||
referrer =
|
||||
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
|
||||
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
|
||||
end
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
|
||||
referrer: referrer,
|
||||
# Only `source` exists in GA4 API
|
||||
utm_source: nil,
|
||||
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
|
||||
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
|
||||
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
|
||||
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
|
||||
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# NOTE: no exits metric in GA4 API currently
|
||||
exits: 0,
|
||||
time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
entry_page: row.dimensions |> Map.fetch!("landingPage"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
# NOTE: no exit pages metrics in GA4 API available for now
|
||||
# defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
|
||||
# %{
|
||||
# site_id: site_id,
|
||||
# import_id: import_id,
|
||||
# date: get_date(row),
|
||||
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
|
||||
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
|
||||
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
|
||||
# }
|
||||
# end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_locations", row) do
|
||||
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
|
||||
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
|
||||
city_data = Location.get_city(city_name, country_code)
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
country: country_code,
|
||||
region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
|
||||
city: city_data && city_data.id,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_devices", row) do
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@browser_google_to_plausible %{
|
||||
"User-Agent:Opera" => "Opera",
|
||||
"Mozilla Compatible Agent" => "Mobile App",
|
||||
"Android Webview" => "Mobile App",
|
||||
"Android Browser" => "Mobile App",
|
||||
"Safari (in-app)" => "Mobile App",
|
||||
"User-Agent: Mozilla" => "Firefox",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_browsers", row) do
|
||||
browser = Map.fetch!(row.dimensions, "browser")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
browser: Map.get(@browser_google_to_plausible, browser, browser),
|
||||
# Does not exist in GA4 API
|
||||
browser_version: nil,
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
@os_google_to_plausible %{
|
||||
"Macintosh" => "Mac",
|
||||
"Linux" => "GNU/Linux",
|
||||
"(not set)" => ""
|
||||
}
|
||||
|
||||
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
|
||||
os = Map.fetch!(row.dimensions, "operatingSystem")
|
||||
|
||||
%{
|
||||
site_id: site_id,
|
||||
import_id: import_id,
|
||||
date: get_date(row),
|
||||
operating_system: Map.get(@os_google_to_plausible, os, os),
|
||||
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
|
||||
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
|
||||
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
|
||||
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
|
||||
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
|
||||
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
|
||||
}
|
||||
end
|
||||
|
||||
defp get_date(%{dimensions: %{"date" => date}}) do
|
||||
date
|
||||
|> Timex.parse!("%Y%m%d", :strftime)
|
||||
|> NaiveDateTime.to_date()
|
||||
end
|
||||
|
||||
defp default_if_missing(value, default \\ nil)
|
||||
defp default_if_missing(value, default) when value in @missing_values, do: default
|
||||
defp default_if_missing(value, _default), do: value
|
||||
|
||||
defp parse_referrer(nil), do: nil
|
||||
defp parse_referrer("(direct)"), do: nil
|
||||
defp parse_referrer("google"), do: "Google"
|
||||
defp parse_referrer("bing"), do: "Bing"
|
||||
defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
|
||||
|
||||
defp parse_referrer(ref) do
|
||||
RefInspector.parse("https://" <> ref)
|
||||
|> PlausibleWeb.RefInspector.parse()
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,35 @@
|
||||
defmodule Plausible.Plugins.API.Capabilities do
|
||||
@moduledoc """
|
||||
Context module for querying API capabilities
|
||||
"""
|
||||
require Plausible.Billing.Feature
|
||||
alias Plausible.Billing.Feature
|
||||
|
||||
@spec get(Plug.Conn.t()) :: {:ok, map()}
|
||||
def get(conn) do
|
||||
conn = PlausibleWeb.Plugs.AuthorizePluginsAPI.call(conn, send_error?: false)
|
||||
|
||||
site = conn.assigns[:authorized_site]
|
||||
|
||||
features =
|
||||
if site do
|
||||
site = Plausible.Repo.preload(site, :owner)
|
||||
|
||||
Feature.list()
|
||||
|> Enum.map(fn mod ->
|
||||
result = mod.check_availability(site.owner)
|
||||
feature = mod |> Module.split() |> List.last()
|
||||
{feature, result == :ok}
|
||||
end)
|
||||
else
|
||||
Enum.map(Feature.list_short_names(), &{&1, false})
|
||||
end
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
authorized: not is_nil(site),
|
||||
data_domain: site && site.domain,
|
||||
features: Enum.into(features, %{})
|
||||
}}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,158 @@
|
||||
defmodule Plausible.S3 do
|
||||
@moduledoc """
|
||||
Helper functions for S3 exports/imports.
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Returns the pre-configured S3 bucket for CSV exports.
|
||||
|
||||
config :plausible, Plausible.S3,
|
||||
exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET")
|
||||
|
||||
Example:
|
||||
|
||||
iex> exports_bucket()
|
||||
"test-exports"
|
||||
|
||||
"""
|
||||
@spec exports_bucket :: String.t()
|
||||
def exports_bucket, do: config(:exports_bucket)
|
||||
|
||||
@doc """
|
||||
Returns the pre-configured S3 bucket for CSV imports.
|
||||
|
||||
config :plausible, Plausible.S3,
|
||||
imports_bucket: System.fetch_env!("S3_IMPORTS_BUCKET")
|
||||
|
||||
Example:
|
||||
|
||||
iex> imports_bucket()
|
||||
"test-imports"
|
||||
|
||||
"""
|
||||
@spec imports_bucket :: String.t()
|
||||
def imports_bucket, do: config(:imports_bucket)
|
||||
|
||||
defp config, do: Application.fetch_env!(:plausible, __MODULE__)
|
||||
defp config(key), do: Keyword.fetch!(config(), key)
|
||||
|
||||
@doc """
|
||||
Presigns an upload for an imported file.
|
||||
|
||||
In the current implementation the bucket always goes into the path component.
|
||||
|
||||
Example:
|
||||
|
||||
iex> %{
|
||||
...> s3_url: "http://localhost:10000/test-imports/123/imported_browsers.csv",
|
||||
...> presigned_url: "http://localhost:10000/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin" <> _
|
||||
...> } = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")
|
||||
|
||||
"""
|
||||
def import_presign_upload(site_id, filename) do
|
||||
config = ExAws.Config.new(:s3)
|
||||
s3_path = Path.join(to_string(site_id), filename)
|
||||
bucket = imports_bucket()
|
||||
{:ok, presigned_url} = ExAws.S3.presigned_url(config, :put, bucket, s3_path)
|
||||
%{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url}
|
||||
end
|
||||
|
||||
# to make ClickHouse see MinIO in dev and test envs we replace
|
||||
# the host in the S3 URL with whatever's set in S3_CLICKHOUSE_HOST env var
|
||||
if Mix.env() in [:dev, :test, :small_dev, :small_test] do
|
||||
defp extract_s3_url(presigned_url) do
|
||||
[s3_url, _] = String.split(presigned_url, "?")
|
||||
|
||||
if ch_host = System.get_env("S3_CLICKHOUSE_HOST") do
|
||||
URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})
|
||||
else
|
||||
s3_url
|
||||
end
|
||||
end
|
||||
else
|
||||
defp extract_s3_url(presigned_url) do
|
||||
[s3_url, _] = String.split(presigned_url, "?")
|
||||
s3_url
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Chunks and uploads Zip archive to the provided S3 destination.
|
||||
|
||||
Returns a presigned URL to download the exported Zip archive from S3.
|
||||
The URL expires in 24 hours.
|
||||
|
||||
In the current implementation the bucket always goes into the path component.
|
||||
"""
|
||||
@spec export_upload_multipart(Enumerable.t(), String.t(), Path.t(), String.t(), keyword) ::
|
||||
:uri_string.uri_string()
|
||||
def export_upload_multipart(stream, s3_bucket, s3_path, filename, config_overrides \\ []) do
|
||||
config = ExAws.Config.new(:s3)
|
||||
|
||||
encoded_filename = URI.encode(filename)
|
||||
disposition = ~s[attachment; filename="#{encoded_filename}"]
|
||||
|
||||
disposition =
|
||||
if encoded_filename != filename do
|
||||
disposition <> "; filename*=utf-8''#{encoded_filename}"
|
||||
else
|
||||
disposition
|
||||
end
|
||||
|
||||
# 5 MiB is the smallest chunk size AWS S3 supports
|
||||
chunk_into_parts(stream, 5 * 1024 * 1024)
|
||||
|> ExAws.S3.upload(s3_bucket, s3_path,
|
||||
content_disposition: disposition,
|
||||
content_type: "application/zip"
|
||||
)
|
||||
|> ExAws.request!(config_overrides)
|
||||
|
||||
{:ok, download_url} =
|
||||
ExAws.S3.presigned_url(config, :get, s3_bucket, s3_path, expires_in: _24hr = 86_400)
|
||||
|
||||
download_url
|
||||
end
|
||||
|
||||
defp chunk_into_parts(stream, min_part_size) do
|
||||
Stream.chunk_while(
|
||||
stream,
|
||||
_acc = %{buffer_size: 0, buffer: [], min_part_size: min_part_size},
|
||||
_chunk_fun = &buffer_until_big_enough/2,
|
||||
_after_fun = &flush_leftovers/1
|
||||
)
|
||||
end
|
||||
|
||||
defp buffer_until_big_enough(data, acc) do
|
||||
%{buffer_size: prev_buffer_size, buffer: prev_buffer, min_part_size: min_part_size} = acc
|
||||
new_buffer_size = prev_buffer_size + IO.iodata_length(data)
|
||||
new_buffer = [prev_buffer | data]
|
||||
|
||||
if new_buffer_size > min_part_size do
|
||||
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
|
||||
{:cont, IO.iodata_to_binary(new_buffer), %{acc | buffer_size: 0, buffer: []}}
|
||||
else
|
||||
{:cont, %{acc | buffer_size: new_buffer_size, buffer: new_buffer}}
|
||||
end
|
||||
end
|
||||
|
||||
defp flush_leftovers(acc) do
|
||||
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
|
||||
{:cont, IO.iodata_to_binary(acc.buffer), %{acc | buffer_size: 0, buffer: []}}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3.
|
||||
|
||||
Example:
|
||||
|
||||
iex> import_clickhouse_credentials()
|
||||
%{access_key_id: "minioadmin", secret_access_key: "minioadmin"}
|
||||
|
||||
"""
|
||||
@spec import_clickhouse_credentials ::
|
||||
%{access_key_id: String.t(), secret_access_key: String.t()}
|
||||
def import_clickhouse_credentials do
|
||||
%{access_key_id: access_key_id, secret_access_key: secret_access_key} = ExAws.Config.new(:s3)
|
||||
%{access_key_id: access_key_id, secret_access_key: secret_access_key}
|
||||
end
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue