Compare commits

..

22 Commits

Author SHA1 Message Date
Adam Rutkowski 9676d3ebd1 PoC - device preparation 3 years ago
Adam Rutkowski 7e8fa24f8f wip 3 years ago
Adam Rutkowski 641e3f636c wip 3 years ago
Adam Rutkowski 5b228d65bc Add debug mode 3 years ago
Adam Rutkowski 4e1ffba5f5 wip 3 years ago
Adam Rutkowski b8f2c14332 wip 3 years ago
Adam Rutkowski 1f1a6d96b7 wip 3 years ago
Adam Rutkowski 0aada2632e wip 3 years ago
Adam Rutkowski 79d427a4a6 wip 3 years ago
Adam Rutkowski 39aef3447b wip 3 years ago
Adam Rutkowski dd19966600 pass 3 years ago
Adam Rutkowski 04d01cc860 wip 3 years ago
Adam Rutkowski b5d6c917a9 wip 3 years ago
Adam Rutkowski d60a18dc12 wip 3 years ago
Adam Rutkowski 00b9213d0c wip 3 years ago
Adam Rutkowski 61cdfd2529 wip 3 years ago
Adam Rutkowski 2fcf5a0bb4 wip 3 years ago
Adam Rutkowski 70063bc79e wip 3 years ago
Adam Rutkowski 1bf2be88d3 satanic 3 years ago
Adam Rutkowski be6aa3ef4c fixup 3 years ago
Adam Rutkowski 1033334279 wip 3 years ago
Adam Rutkowski 0f95999eb1 Proof of concept 3 years ago

@ -2,5 +2,11 @@
plugins: [Phoenix.LiveView.HTMLFormatter], plugins: [Phoenix.LiveView.HTMLFormatter],
import_deps: [:ecto, :ecto_sql, :phoenix], import_deps: [:ecto, :ecto_sql, :phoenix],
subdirectories: ["priv/*/migrations"], subdirectories: ["priv/*/migrations"],
inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] inputs: ["*.{heex,ex,exs}", "{config,lib,test,extra}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"],
locals_without_parens: [
pageview: 1,
pageview: 2,
custom_event: 1,
custom_event: 2
]
] ]

@ -1,42 +1,26 @@
name: Elixir CI name: Elixir CI
on: on:
pull_request:
push: push:
branches: [master, stable] branches: [master, stable]
pull_request:
concurrency: concurrency:
group: ${{ github.workflow }}-${{ github.ref }} group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
env:
CACHE_VERSION: v7
PERSISTENT_CACHE_DIR: cached
jobs: jobs:
build: build:
name: "Build and test (${{ matrix.mix_env }}, ${{ matrix.postgres_image }}${{ matrix.test_experimental_reduced_joins == '1' && ', experimental_reduced_joins' || '' }})" name: Build and test
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
mix_env: ['test', 'small_test'] postgres_version: [15, 16]
postgres_image: ['postgres:16']
test_experimental_reduced_joins: ['0']
include:
- mix_env: 'test'
postgres_image: 'postgres:15'
test_experimental_reduced_joins: '0'
- mix_env: 'test'
postgres_image: 'postgres:16'
test_experimental_reduced_joins: '1'
env:
MIX_ENV: ${{ matrix.mix_env }}
TEST_EXPERIMENTAL_REDUCED_JOINS: ${{ matrix.test_experimental_reduced_joins }}
services: services:
postgres: postgres:
image: ${{ matrix.postgres_image }} image: postgres:${{ matrix.postgres_version }}
ports: ports:
- 5432:5432 - 5432:5432
env: env:
@ -46,6 +30,7 @@ jobs:
--health-interval 10s --health-interval 10s
--health-timeout 5s --health-timeout 5s
--health-retries 5 --health-retries 5
clickhouse: clickhouse:
image: clickhouse/clickhouse-server:23.3.7.5-alpine image: clickhouse/clickhouse-server:23.3.7.5-alpine
ports: ports:
@ -56,91 +41,43 @@ jobs:
--health-interval 10s --health-interval 10s
--health-timeout 5s --health-timeout 5s
--health-retries 5 --health-retries 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: marocchino/tool-versions-action@v1
id: versions
- uses: erlef/setup-beam@v1
with:
elixir-version: ${{ steps.versions.outputs.elixir }}
otp-version: ${{ steps.versions.outputs.erlang }}
- uses: actions/cache@v4
with:
path: |
deps
_build
tracker/node_modules
priv/tracker/js
${{ env.PERSISTENT_CACHE_DIR }}
key: ${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
restore-keys: |
${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
- name: Check for changes in tracker/**
uses: dorny/paths-filter@v3
id: changes
with:
filters: |
tracker:
- 'tracker/**'
- name: Check if priv/tracker/js/plausible.js exists
run: |
if [ -f priv/tracker/js/plausible.js ]; then
echo "HAS_BUILT_TRACKER=true" >> $GITHUB_ENV
else
echo "HAS_BUILT_TRACKER=false" >> $GITHUB_ENV
fi
- run: npm install --prefix ./tracker
if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
- run: npm run deploy --prefix ./tracker
if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
- run: mix deps.get --only $MIX_ENV
- run: mix compile --warnings-as-errors --all-warnings
- run: mix do ecto.create, ecto.migrate
- run: mix run -e "Tzdata.ReleaseUpdater.poll_for_update"
- run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors
if: env.MIX_ENV == 'test'
- run: mix test --include slow --max-failures 1 --warnings-as-errors
if: env.MIX_ENV == 'small_test'
static:
name: Static checks (format, credo, dialyzer)
env:
MIX_ENV: test
runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Read .tool-versions
- uses: marocchino/tool-versions-action@v1 uses: marocchino/tool-versions-action@v1
id: versions id: versions
- uses: erlef/setup-beam@v1 - name: Set up Elixir
uses: erlef/setup-beam@v1
with: with:
elixir-version: ${{ steps.versions.outputs.elixir }} elixir-version: ${{steps.versions.outputs.elixir}}
otp-version: ${{ steps.versions.outputs.erlang }} otp-version: ${{ steps.versions.outputs.erlang}}
- name: Restore dependencies cache
- uses: actions/cache@v4 uses: buildjet/cache@v4
with: with:
path: | path: |
deps deps
_build _build
priv/plts priv/plts
key: static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }} key: ${{ runner.os }}-mix-v7-${{ hashFiles('**/mix.lock') }}
restore-keys: | restore-keys: ${{ runner.os }}-mix-v7-
static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}- - name: Install dependencies
static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master- run: mix deps.get && npm install --prefix ./tracker
- name: Check Formatting
- run: mix deps.get run: mix format --check-formatted
- run: mix compile --warnings-as-errors --all-warnings - name: Check Compile Warnings
- run: mix format --check-formatted run: mix compile --warnings-as-errors --all-warnings
- run: mix deps.unlock --check-unused - name: Generate tracker files
- run: mix credo diff --from-git-merge-base origin/master run: npm run deploy --prefix ./tracker
- run: mix dialyzer - name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- name: Run tests
run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors
- name: Run tests (small build)
run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors
- name: Check Dialyzer
run: mix dialyzer
env:
MIX_ENV: test

@ -1,28 +0,0 @@
name: Validate migrations
on:
pull_request:
paths:
- 'priv/repo/migrations/**'
- 'priv/ingest_repo/migrations/**'
jobs:
validate:
name: App code does not change at the same time
runs-on: ubuntu-latest
steps:
- uses: dorny/paths-filter@v3
id: changes
with:
list-files: json
filters: |
lib:
- 'lib/**'
- 'extra/**'
- 'config/**'
- if: steps.changes.outputs.lib == 'true'
run: |
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
exit 1

3
.gitignore vendored

@ -19,9 +19,6 @@ erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build"). # Also ignore archive artifacts (built via "mix archive.build").
*.ez *.ez
# Temporary files, for example, from tests.
/tmp/
# Ignore package tarball (built via "mix hex.build"). # Ignore package tarball (built via "mix hex.build").
plausible-*.tar plausible-*.tar

@ -2,12 +2,6 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
### Added ### Added
- Pages Block List in Site Settings
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export
- Ability to display total conversions (with a goal filter) on the main graph
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `time_on_page` metric into the Stats API
- County Block List in Site Settings - County Block List in Site Settings
- Query the `views_per_visit` metric based on imported data as well if possible - Query the `views_per_visit` metric based on imported data as well if possible
- Group `operating_system_versions` by `operating_system` in Stats API breakdown - Group `operating_system_versions` by `operating_system` in Stats API breakdown
@ -36,19 +30,14 @@ All notable changes to this project will be documented in this file.
- Add 'browser_versions.csv' to CSV export - Add 'browser_versions.csv' to CSV export
- Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB) - Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB)
- Add alternative SMTP adapter plausible/analytics#3654 - Add alternative SMTP adapter plausible/analytics#3654
- Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906
- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
### Removed ### Removed
- Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions
- Removed the `prop_names` returned in the Stats API `event:goal` breakdown response - Removed the `prop_names` returned in the Stats API `event:goal` breakdown response
- Removed the `prop-breakdown.csv` file from CSV export - Removed the `prop-breakdown.csv` file from CSV export
- Deprecated `CLICKHOUSE_MAX_BUFFER_SIZE` - Deprecated `CLICKHOUSE_MAX_BUFFER_SIZE`
- Removed `/app/init-admin.sh` that was deprecated in v2.0.0 plausible/analytics#3903
- Remove `DISABLE_AUTH` deprecation warning plausible/analytics#3904
### Changed ### Changed
- A visits `entry_page` and `exit_page` is only set and updated for pageviews, not custom events
- Limit the number of Goal Conversions shown on the dashboard and render a "Details" link when there are more entries to show - Limit the number of Goal Conversions shown on the dashboard and render a "Details" link when there are more entries to show
- Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal - Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal
- Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard - Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard
@ -57,8 +46,6 @@ All notable changes to this project will be documented in this file.
- Validate metric isn't queried multiple times - Validate metric isn't queried multiple times
### Fixed ### Fixed
- Creating many sites no longer leads to cookie overflow
- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns
- Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown - Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown
- Fix `conversion_rate` metric in a `browser_versions` breakdown - Fix `conversion_rate` metric in a `browser_versions` breakdown
- Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division) - Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division)
@ -77,10 +64,7 @@ All notable changes to this project will be documented in this file.
- Allow custom event timeseries in stats API plausible/analytics#3505 - Allow custom event timeseries in stats API plausible/analytics#3505
- Fixes for sites with UTF characters in domain plausible/analytics#3560 - Fixes for sites with UTF characters in domain plausible/analytics#3560
- Fix crash when using special characters in filter plausible/analytics#3634 - Fix crash when using special characters in filter plausible/analytics#3634
- Fix automatic scrolling to the bottom on the dashboard if previously selected properties tab plausible/analytics#3872
- Allow running the container with arbitrary UID plausible/analytics#2986 - Allow running the container with arbitrary UID plausible/analytics#2986
- Fix `width=manual` in embedded dashboards plausible/analytics#3910
- Fix URL escaping when pipes are used in UTM tags plausible/analytics#3930
## v2.0.0 - 2023-07-12 ## v2.0.0 - 2023-07-12

@ -47,7 +47,8 @@ RUN npm run deploy --prefix ./tracker && \
mix assets.deploy && \ mix assets.deploy && \
mix phx.digest priv/static && \ mix phx.digest priv/static && \
mix download_country_database && \ mix download_country_database && \
mix sentry.package_source_code # https://hexdocs.pm/sentry/Sentry.Sources.html#module-source-code-storage
mix sentry_recompile
WORKDIR /app WORKDIR /app
COPY rel rel COPY rel rel

@ -38,10 +38,7 @@ postgres-stop: ## Stop and remove the postgres container
docker stop plausible_db && docker rm plausible_db docker stop plausible_db && docker rm plausible_db
minio: ## Start a transient container with a recent version of minio (s3) minio: ## Start a transient container with a recent version of minio (s3)
docker run -d --rm -p 10000:10000 -p 10001:10001 --name plausible_minio minio/minio server /data --address ":10000" --console-address ":10001" docker run -d --rm -p 6000:6000 -p 6001:6001 --name plausible_minio minio/minio server /data --address ":6000" --console-address ":6001"
while ! docker exec plausible_minio mc alias set local http://localhost:10000 minioadmin minioadmin; do sleep 1; done
docker exec plausible_minio mc mb local/dev-exports
docker exec plausible_minio mc mb local/dev-imports
minio-stop: ## Stop and remove the minio container minio-stop: ## Stop and remove the minio container
docker stop plausible_minio docker stop plausible_minio

@ -22,7 +22,6 @@ if (container) {
conversionsOptedOut: container.dataset.conversionsOptedOut === 'true', conversionsOptedOut: container.dataset.conversionsOptedOut === 'true',
funnelsOptedOut: container.dataset.funnelsOptedOut === 'true', funnelsOptedOut: container.dataset.funnelsOptedOut === 'true',
propsOptedOut: container.dataset.propsOptedOut === 'true', propsOptedOut: container.dataset.propsOptedOut === 'true',
revenueGoals: JSON.parse(container.dataset.revenueGoals),
funnels: JSON.parse(container.dataset.funnels), funnels: JSON.parse(container.dataset.funnels),
statsBegin: container.dataset.statsBegin, statsBegin: container.dataset.statsBegin,
nativeStatsBegin: container.dataset.nativeStatsBegin, nativeStatsBegin: container.dataset.nativeStatsBegin,

@ -159,10 +159,10 @@ export default function PlausibleCombobox(props) {
}, []) }, [])
useEffect(() => { useEffect(() => {
if (props.singleOption && props.values.length === 0 && props.autoFocus) { if (props.singleOption && props.values.length === 0) {
searchRef.current.focus() searchRef.current.focus()
} }
}, [props.values.length === 0, props.singleOption, props.autoFocus]) }, [props.values.length === 0])
const searchBoxClass = 'border-none py-1 px-0 w-full inline-block rounded-md focus:outline-none focus:ring-0 text-sm' const searchBoxClass = 'border-none py-1 px-0 w-full inline-block rounded-md focus:outline-none focus:ring-0 text-sm'

@ -258,10 +258,6 @@ class Filters extends React.Component {
) )
} }
trackFilterMenu() {
window.plausible && window.plausible('Filter Menu: Open', {u: `${window.location.protocol}//${window.location.hostname}/:dashboard`})
}
renderDropDown() { renderDropDown() {
const { history, query, site } = this.props; const { history, query, site } = this.props;
@ -270,7 +266,7 @@ class Filters extends React.Component {
{({ open }) => ( {({ open }) => (
<> <>
<div> <div>
<Menu.Button onClick={this.trackFilterMenu} className="flex items-center text-xs md:text-sm font-medium leading-tight px-3 py-2 cursor-pointer ml-auto text-gray-500 dark:text-gray-200 hover:bg-gray-200 dark:hover:bg-gray-900 rounded"> <Menu.Button className="flex items-center text-xs md:text-sm font-medium leading-tight px-3 py-2 cursor-pointer ml-auto text-gray-500 dark:text-gray-200 hover:bg-gray-200 dark:hover:bg-gray-900 rounded">
{this.renderDropdownButton()} {this.renderDropdownButton()}
</Menu.Button> </Menu.Button>
</div> </div>

@ -59,7 +59,6 @@ export function parseQuery(querystring, site) {
'region': q.get('region'), 'region': q.get('region'),
'city': q.get('city'), 'city': q.get('city'),
'page': q.get('page'), 'page': q.get('page'),
'hostname': q.get('hostname'),
'entry_page': q.get('entry_page'), 'entry_page': q.get('entry_page'),
'exit_page': q.get('exit_page') 'exit_page': q.get('exit_page')
} }

@ -1,53 +1,27 @@
import numberFormatter, {durationFormatter} from '../../util/number-formatter' import numberFormatter, {durationFormatter} from '../../util/number-formatter'
import { parsePrefix } from '../../util/filters'
export function getGraphableMetrics(query, site) { export const METRIC_MAPPING = {
const isRealtime = query.period === 'realtime' 'Unique visitors (last 30 min)': 'visitors',
const goalFilter = query.filters.goal 'Pageviews (last 30 min)': 'pageviews',
const pageFilter = query.filters.page 'Unique visitors': 'visitors',
'Visit duration': 'visit_duration',
if (isRealtime && goalFilter) { 'Total pageviews': 'pageviews',
return ["visitors"] 'Views per visit': 'views_per_visit',
} else if (isRealtime) { 'Total visits': 'visits',
return ["visitors", "pageviews"] 'Bounce rate': 'bounce_rate',
} else if (goalFilter && canGraphRevenueMetrics(goalFilter, site)) { 'Unique conversions': 'conversions',
return ["visitors", "events", "average_revenue", "total_revenue", "conversion_rate"] 'Average revenue': 'average_revenue',
} else if (goalFilter) { 'Total revenue': 'total_revenue',
return ["visitors", "events", "conversion_rate"]
} else if (pageFilter) {
return ["visitors", "visits", "pageviews", "bounce_rate", "time_on_page"]
} else {
return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]
}
}
// Revenue metrics can only be graphed if:
// * The query is filtered by at least one revenue goal
// * All revenue goals in filter have the same currency
function canGraphRevenueMetrics(goalFilter, site) {
const goalsInFilter = parsePrefix(goalFilter).values
const revenueGoalsInFilter = site.revenueGoals.filter((rg) => {
return goalsInFilter.includes(rg.event_name)
})
const singleCurrency = revenueGoalsInFilter.every((rg) => {
return rg.currency === revenueGoalsInFilter[0].currency
})
return revenueGoalsInFilter.length > 0 && singleCurrency
} }
export const METRIC_LABELS = { export const METRIC_LABELS = {
'visitors': 'Visitors', 'visitors': 'Visitors',
'pageviews': 'Pageviews', 'pageviews': 'Pageviews',
'events': 'Total Conversions',
'views_per_visit': 'Views per Visit', 'views_per_visit': 'Views per Visit',
'visits': 'Visits', 'visits': 'Visits',
'bounce_rate': 'Bounce Rate', 'bounce_rate': 'Bounce Rate',
'visit_duration': 'Visit Duration', 'visit_duration': 'Visit Duration',
'conversions': 'Converted Visitors', 'conversions': 'Converted Visitors',
'conversion_rate': 'Conversion Rate',
'average_revenue': 'Average Revenue', 'average_revenue': 'Average Revenue',
'total_revenue': 'Total Revenue', 'total_revenue': 'Total Revenue',
} }
@ -55,17 +29,23 @@ export const METRIC_LABELS = {
export const METRIC_FORMATTER = { export const METRIC_FORMATTER = {
'visitors': numberFormatter, 'visitors': numberFormatter,
'pageviews': numberFormatter, 'pageviews': numberFormatter,
'events': numberFormatter,
'visits': numberFormatter, 'visits': numberFormatter,
'views_per_visit': (number) => (number), 'views_per_visit': (number) => (number),
'bounce_rate': (number) => (`${number}%`), 'bounce_rate': (number) => (`${number}%`),
'visit_duration': durationFormatter, 'visit_duration': durationFormatter,
'conversions': numberFormatter, 'conversions': numberFormatter,
'conversion_rate': (number) => (`${number}%`),
'total_revenue': numberFormatter, 'total_revenue': numberFormatter,
'average_revenue': numberFormatter, 'average_revenue': numberFormatter,
} }
export const LoadingState = {
loading: 'loading',
refreshing: 'refreshing',
loaded: 'loaded',
isLoadingOrRefreshing: function (state) { return [this.loading, this.refreshing].includes(state) },
isLoadedOrRefreshing: function (state) { return [this.loaded, this.refreshing].includes(state) }
}
const buildComparisonDataset = function(comparisonPlot) { const buildComparisonDataset = function(comparisonPlot) {
if (!comparisonPlot) return [] if (!comparisonPlot) return []

@ -6,7 +6,7 @@ import * as storage from '../../util/storage'
import { isKeyPressed } from '../../keybinding.js' import { isKeyPressed } from '../../keybinding.js'
import { monthsBetweenDates } from '../../util/date.js' import { monthsBetweenDates } from '../../util/date.js'
const INTERVAL_LABELS = { export const INTERVAL_LABELS = {
'minute': 'Minutes', 'minute': 'Minutes',
'hour': 'Hours', 'hour': 'Hours',
'date': 'Days', 'date': 'Days',
@ -14,19 +14,11 @@ const INTERVAL_LABELS = {
'month': 'Months' 'month': 'Months'
} }
function validIntervals(site, query) { export const getStoredInterval = function(period, domain) {
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
return ["week", "month"]
} else {
return site.validIntervalsByPeriod[query.period]
}
}
function getStoredInterval(period, domain) {
return storage.getItem(`interval__${period}__${domain}`) return storage.getItem(`interval__${period}__${domain}`)
} }
function storeInterval(period, domain, interval) { export const storeInterval = function(period, domain, interval) {
storage.setItem(`interval__${period}__${domain}`, interval) storage.setItem(`interval__${period}__${domain}`, interval)
} }
@ -41,47 +33,37 @@ function subscribeKeybinding(element) {
}, [handleKeyPress]) }, [handleKeyPress])
} }
export const getCurrentInterval = function(site, query) { function DropdownItem({ option, currentInterval, updateInterval }) {
const options = validIntervals(site, query) return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
const storedInterval = getStoredInterval(query.period, site.domain) {({ active }) => (
const defaultInterval = [...options].pop() <span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
if (storedInterval && options.includes(storedInterval)) { 'text-gray-700 dark:text-gray-200': !active,
return storedInterval 'font-bold cursor-none select-none': option == currentInterval,
} else { }, 'block px-4 py-2 text-sm')}>
return defaultInterval {INTERVAL_LABELS[option]}
} </span>
)}
</Menu.Item>
)
} }
export function IntervalPicker({ query, site, onIntervalUpdate }) { export function IntervalPicker({ graphData, query, site, updateInterval }) {
if (query.period == 'realtime') return null if (query.period == 'realtime') return null
const menuElement = React.useRef(null)
const options = validIntervals(site, query)
const currentInterval = getCurrentInterval(site, query)
const menuElement = React.useRef(null)
subscribeKeybinding(menuElement) subscribeKeybinding(menuElement)
function updateInterval(interval) { let currentInterval = graphData?.interval
storeInterval(query.period, site.domain, interval)
onIntervalUpdate(interval) let options = site.validIntervalsByPeriod[query.period]
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
options = ["week", "month"]
} }
function renderDropdownItem(option) { if (!options.includes(currentInterval)) {
return ( currentInterval = [...options].pop()
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
} }
return ( return (
@ -103,7 +85,7 @@ export function IntervalPicker({ query, site, onIntervalUpdate }) {
leaveFrom="opacity-100 scale-100" leaveFrom="opacity-100 scale-100"
leaveTo="opacity-0 scale-95"> leaveTo="opacity-0 scale-95">
<Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static> <Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static>
{options.map(renderDropdownItem)} {options.map((option) => DropdownItem({ option, currentInterval, updateInterval }))}
</Menu.Items> </Menu.Items>
</Transition> </Transition>
</> </>

@ -1,247 +0,0 @@
import React from 'react';
import { withRouter } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
import dateFormatter from './date-formatter';
import FadeIn from '../../fade-in';
import classNames from 'classnames';
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
}
getGraphMetric() {
let metric = this.props.graphData.metric
if (metric == 'visitors' && this.props.query.filters.goal) {
return 'conversions'
} else {
return metric
}
}
regenerateChart() {
const { graphData, query } = this.props
const metric = this.getGraphMetric()
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.maybeHopToHoveredPeriod.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
window.addEventListener('mousemove', this.repositionTooltip);
}
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
}
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
maybeHopToHoveredPeriod(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
render() {
const { graphData } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
)
}
}
export default withRouter(LineGraph)

@ -1,17 +0,0 @@
import React from "react"
export default function SamplingNotice({topStatData}) {
const samplePercent = topStatData?.samplePercent
if (samplePercent && samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
} else {
return null
}
}

@ -1,53 +0,0 @@
import React, { useState } from "react"
import * as api from '../../api'
import { getCurrentInterval } from "./interval-picker"
export default function StatsExport({site, query}) {
const [exporting, setExporting] = useState(false)
function startExport() {
setExporting(true)
document.cookie = "exporting="
pollExportReady()
}
function pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(pollExportReady, 1000)
} else {
setExporting(false)
}
}
function renderLoading() {
return (
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
)
}
function renderExportLink() {
const interval = getCurrentInterval(site, query)
const queryParams = api.serializeQuery(query, [{ interval }])
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
return (
<a href={endpoint} download onClick={startExport}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
<polyline points="7 10 12 15 17 10"></polyline>
<line x1="12" y1="15" x2="12" y2="3"></line>
</svg>
</a>
)
}
return (
<div className="w-4 h-4 mx-2">
{exporting && renderLoading()}
{!exporting && renderExportLink()}
</div>
)
}

@ -3,9 +3,8 @@ import { Tooltip } from '../../util/tooltip'
import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load' import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load'
import classNames from "classnames"; import classNames from "classnames";
import numberFormatter, { durationFormatter } from '../../util/number-formatter' import numberFormatter, { durationFormatter } from '../../util/number-formatter'
import * as storage from '../../util/storage' import { METRIC_MAPPING } from './graph-util'
import { formatDateRange } from '../../util/date.js' import { formatDateRange } from '../../util/date.js'
import { getGraphableMetrics } from "./graph-util.js";
function Maybe({condition, children}) { function Maybe({condition, children}) {
if (condition) { if (condition) {
@ -15,103 +14,99 @@ function Maybe({condition, children}) {
} }
} }
function renderPercentageComparison(name, comparison, forceDarkBg = false) { export default class TopStats extends React.Component {
const formattedComparison = numberFormatter(Math.abs(comparison)) renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg, const defaultClassName = classNames({
"pl-2 text-xs text-gray-100": forceDarkBg "pl-2 text-xs dark:text-gray-100": !forceDarkBg,
}) "pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg, const noChangeClassName = classNames({
"pl-2 text-xs text-gray-300": forceDarkBg "pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
}) "pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500' if (comparison > 0) {
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span> const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
} else if (comparison < 0) { return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400' } else if (comparison < 0) {
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span> const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
} else if (comparison === 0) { return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
return <span className={noChangeClassName}>&#12336; 0%</span> } else if (comparison === 0) {
} else { return <span className={noChangeClassName}>&#12336; 0%</span>
return null } else {
return null
}
} }
}
function topStatNumberShort(name, value) { topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) { if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value) return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) { } else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%' return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) { } else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short return value?.short
} else { } else {
return numberFormatter(value) return numberFormatter(value)
}
} }
}
function topStatNumberLong(name, value) { topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) { if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value) return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) { } else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%' return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) { } else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long return value?.long
} else { } else {
return (value || 0).toLocaleString() return (value || 0).toLocaleString()
}
} }
}
export default function TopStats(props) {
const {site, query, data, onMetricUpdate, tooltipBoundary, lastLoadTimestamp} = props
function tooltip(stat) { topStatTooltip(stat, query) {
let statName = stat.name.toLowerCase() let statName = stat.name.toLowerCase()
statName = stat.value === 1 ? statName.slice(0, -1) : statName statName = stat.value === 1 ? statName.slice(0, -1) : statName
return ( return (
<div> <div>
{query.comparison && <div className="whitespace-nowrap"> {query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} vs. {topStatNumberLong(stat.name, stat.comparison_value)} {statName} {this.topStatNumberLong(stat.name, stat.value)} vs. {this.topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{renderPercentageComparison(stat.name, stat.change, true)}</span> <span className="ml-2">{this.renderPercentageComparison(stat.name, stat.change, true)}</span>
</div>} </div>}
{!query.comparison && <div className="whitespace-nowrap"> {!query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} {statName} {this.topStatNumberLong(stat.name, stat.value)} {statName}
</div>} </div>}
{stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp}/>s ago</p>} {stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={this.props.lastLoadTimestamp}/>s ago</p>}
</div> </div>
) )
} }
function canMetricBeGraphed(stat) { canMetricBeGraphed(stat) {
const graphableMetrics = getGraphableMetrics(query, site) const isTotalUniqueVisitors = this.props.query.filters.goal && stat.name === 'Unique visitors'
return stat.graph_metric && graphableMetrics.includes(stat.graph_metric) const isKnownMetric = Object.keys(METRIC_MAPPING).includes(stat.name)
return isKnownMetric && !isTotalUniqueVisitors
} }
function maybeUpdateMetric(stat) { maybeUpdateMetric(stat) {
if (canMetricBeGraphed(stat)) { if (this.canMetricBeGraphed(stat)) {
storage.setItem(`metric__${site.domain}`, stat.graph_metric) this.props.updateMetric(METRIC_MAPPING[stat.name])
onMetricUpdate(stat.graph_metric)
} }
} }
function blinkingDot() { blinkingDot() {
return ( return (
<div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div> <div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div>
) )
} }
function getStoredMetric() { renderStatName(stat) {
return storage.getItem(`metric__${site.domain}`) const { metric } = this.props
} const isSelected = metric === METRIC_MAPPING[stat.name]
function renderStatName(stat) {
const isSelected = stat.graph_metric === getStoredMetric()
const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g) const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g)
@ -128,45 +123,48 @@ export default function TopStats(props) {
) )
} }
function renderStat(stat, index) { render() {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', { const { topStatData, query, site } = this.props
'cursor-pointer': canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0, const stats = topStatData && topStatData.top_stats.map((stat, index) => {
'border-r lg:border-r-0': index % 2 === 0
}) const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': this.canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
return (
<Tooltip key={stat.name} info={this.topStatTooltip(stat, query)} className={className} onClick={() => { this.maybeUpdateMetric(stat) }} boundary={this.props.tooltipBoundary}>
{this.renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={METRIC_MAPPING[stat.name]}>{this.topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ this.renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, topStatData.from, topStatData.to) }</p>
</Maybe>
</div>
return (
<Tooltip key={stat.name} info={tooltip(stat, query)} className={className} onClick={() => { maybeUpdateMetric(stat) }} boundary={tooltipBoundary}>
{renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={stat.graph_metric}>{topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}> <Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, data.from, data.to) }</p> <div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ this.topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, topStatData.comparing_from, topStatData.comparing_to) }</p>
</div>
</Maybe> </Maybe>
</div>
<Maybe condition={query.comparison}>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, data.comparing_from, data.comparing_to) }</p>
</div> </div>
</Maybe> </Tooltip>
</div> )
</Tooltip> })
)
}
const stats = data && data.top_stats.map(renderStat) if (stats && query && query.period === 'realtime') {
stats.push(this.blinkingDot())
}
if (stats && query.period === 'realtime') { return stats || null;
stats.push(blinkingDot())
} }
return stats || null;
} }

@ -1,153 +1,534 @@
import React, { useState, useEffect, useRef, useCallback } from 'react'; import React from 'react';
import { withRouter, Link } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import * as api from '../../api' import * as api from '../../api'
import * as storage from '../../util/storage' import * as storage from '../../util/storage'
import { getGraphableMetrics } from './graph-util' import LazyLoader from '../../components/lazy-loader'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_MAPPING, METRIC_LABELS, METRIC_FORMATTER, LoadingState } from './graph-util'
import dateFormatter from './date-formatter';
import TopStats from './top-stats'; import TopStats from './top-stats';
import { IntervalPicker, getCurrentInterval } from './interval-picker' import { IntervalPicker, getStoredInterval, storeInterval } from './interval-picker';
import StatsExport from './stats-export'
import WithImportedSwitch from './with-imported-switch';
import SamplingNotice from './sampling-notice';
import FadeIn from '../../fade-in'; import FadeIn from '../../fade-in';
import * as url from '../../util/url' import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input' import { isComparisonEnabled } from '../../comparison-input'
import LineGraphWithRouter from './line-graph'
function fetchTopStats(site, query) { const calculateMaximumY = function(dataset) {
const q = { ...query } const yAxisValues = dataset
.flatMap((item) => item.data)
if (!isComparisonEnabled(q.comparison)) { .map((item) => item || 0)
q.comparison = 'previous_period'
}
return api.get(url.apiPath(site, '/top-stats'), q) if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
} }
function fetchMainGraph(site, query, metric, interval) { class LineGraph extends React.Component {
const params = {metric, interval} constructor(props) {
return api.get(url.apiPath(site, '/main-graph'), query, params) super(props);
} this.boundary = React.createRef()
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
this.state = {
exported: false
};
}
export default function VisitorGraph(props) { regenerateChart() {
const {site, query, lastLoadTimestamp} = props const { graphData, metric, query } = this.props
const isRealtime = query.period === 'realtime' const graphEl = document.getElementById("main-graph-canvas")
const isDarkTheme = document.querySelector('html').classList.contains('dark') || false this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
const topStatsBoundary = useRef(null) return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.onClick.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const [topStatData, setTopStatData] = useState(null) const hasMultipleYears =
const [topStatsLoading, setTopStatsLoading] = useState(true) graphData.labels
const [graphData, setGraphData] = useState(null) .filter((date) => typeof date === 'string')
const [graphLoading, setGraphLoading] = useState(true) .map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
// This state is explicitly meant for the situation where either graph interval if (graphData.interval === 'hour' && query.period !== 'day') {
// or graph metric is changed. That results in behaviour where Top Stats stay const date = dateFormatter({
// intact, but the graph container alone will display a loading spinner for as interval: "date",
// long as new graph data is fetched. longForm: false,
const [graphRefreshing, setGraphRefreshing] = useState(false) period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const onIntervalUpdate = useCallback((newInterval) => { // Returns a combination of date and hour. This is because
setGraphData(null) // small intervals like hour may return multiple days
setGraphRefreshing(true) // depending on the query period.
fetchGraphData(getStoredMetric(), newInterval) return `${date}, ${hour}`
}, [query]) }
const onMetricUpdate = useCallback((newMetric) => { if (graphData.interval === 'minute' && query.period !== 'realtime') {
setGraphData(null) return dateFormatter({
setGraphRefreshing(true) interval: "hour", longForm: false, period: query.period,
fetchGraphData(newMetric, getCurrentInterval(site, query)) })(this.getLabelForValue(val))
}, [query]) }
useEffect(() => { return dateFormatter({
setTopStatData(null) interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
setTopStatsLoading(true) })(this.getLabelForValue(val))
setGraphData(null) },
setGraphLoading(true) color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
fetchTopStatsAndGraphData() }
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
if (isRealtime) { repositionTooltip(e) {
document.addEventListener('tick', fetchTopStatsAndGraphData) const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
} }
}
return () => { componentDidMount() {
document.removeEventListener('tick', fetchTopStatsAndGraphData) if (this.props.graphData) {
this.chart = this.regenerateChart();
} }
}, [query]) window.addEventListener('mousemove', this.repositionTooltip);
}
useEffect(() => { componentDidUpdate(prevProps) {
if (topStatData) { storeTopStatsContainerHeight() } const { graphData, darkTheme } = this.props;
}, [topStatData]) const tooltip = document.getElementById('chartjs-tooltip');
function fetchTopStatsAndGraphData() { if (
fetchTopStats(site, query) graphData !== prevProps.graphData ||
.then((res) => { darkTheme !== prevProps.darkTheme
setTopStatData(res) ) {
setTopStatsLoading(false)
}) if (graphData) {
if (this.chart) {
let metric = getStoredMetric() this.chart.destroy();
const availableMetrics = getGraphableMetrics(query, site) }
this.chart = this.regenerateChart();
if (!availableMetrics.includes(metric)) { this.chart.update();
metric = availableMetrics[0] }
storage.setItem(`metric__${site.domain}`, metric)
if (tooltip) {
tooltip.style.display = 'none';
}
} }
const interval = getCurrentInterval(site, query) if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
fetchGraphData(metric, interval) if (tooltip) {
tooltip.style.display = 'none';
}
}
} }
function fetchGraphData(metric, interval) { componentWillUnmount() {
fetchMainGraph(site, query, metric, interval) // Ensure that the tooltip doesn't hang around when we are loading more data
.then((res) => { const tooltip = document.getElementById('chartjs-tooltip');
setGraphData(res) if (tooltip) {
setGraphLoading(false) tooltip.style.opacity = 0;
setGraphRefreshing(false) tooltip.style.display = 'none';
}) }
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
onClick(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(this.pollExportReady.bind(this), 1000);
} else {
this.setState({ exported: false })
}
}
downloadSpinner() {
this.setState({ exported: true });
document.cookie = "exporting=";
setTimeout(this.pollExportReady.bind(this), 1000);
}
downloadLink() {
if (this.props.query.period !== 'realtime') {
if (this.state.exported) {
return (
<div className="w-4 h-4 mx-2">
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
</div>
)
} else {
const interval = this.props.graphData?.interval
const queryParams = api.serializeQuery(this.props.query, [{ interval }])
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${queryParams}`
return (
<a className="w-4 h-4 mx-2" href={endpoint} download onClick={this.downloadSpinner.bind(this)}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
</a>
)
}
}
} }
function getStoredMetric() { samplingNotice() {
return storage.getItem(`metric__${site.domain}`) const samplePercent = this.props.topStatData && this.props.topStatData.sample_percent
if (samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
}
} }
function storeTopStatsContainerHeight() { importedNotice() {
storage.setItem(`topStatsHeight__${site.domain}`, document.getElementById('top-stats-container').clientHeight) if (!this.props.topStatData?.imported_source) return
const isBeforeNativeStats = (date) => {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(this.props.site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(this.props.topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
const strike = withImported ? "" : " line-through"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
<svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
</svg>
</div>
</Link>
)
}
} }
// This function is used for maintaining the main-graph/top-stats container height in the // This function is used for maintaining the main-graph/top-stats container height in the
// loading process. The container height depends on how many top stat metrics are returned // loading process. The container height depends on how many top stat metrics are returned
// from the API, but in the loading state, we don't know that yet. We can use localStorage // from the API, but in the loading state, we don't know that yet. We can use localStorage
// to keep track of the Top Stats container height. // to keep track of the Top Stats container height.
function getTopStatsHeight() { getTopStatsHeight() {
if (topStatData) { if (this.props.topStatData) {
return 'auto' return 'auto'
} else { } else {
return `${storage.getItem(`topStatsHeight__${site.domain}`) || 89}px` return `${storage.getItem(`topStatsHeight__${this.props.site.domain}`) || 89}px`
} }
} }
render() {
const { mainGraphRefreshing, updateMetric, updateInterval, metric, topStatData, query, site, graphData, lastLoadTimestamp } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return ( return (
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}> <div>
{(topStatsLoading || graphLoading) && renderLoader()} <div id="top-stats-container" className="flex flex-wrap" ref={this.boundary} style={{ height: this.getTopStatsHeight() }}>
<FadeIn show={!(topStatsLoading || graphLoading)}> <TopStats site={site} query={query} metric={metric} updateMetric={updateMetric} topStatData={topStatData} tooltipBoundary={this.boundary.current} lastLoadTimestamp={lastLoadTimestamp} />
<div id="top-stats-container" className="flex flex-wrap" ref={topStatsBoundary} style={{ height: getTopStatsHeight() }}>
<TopStats site={site} query={query} data={topStatData} onMetricUpdate={onMetricUpdate} tooltipBoundary={topStatsBoundary.current} lastLoadTimestamp={lastLoadTimestamp} />
</div> </div>
<div className="relative px-2"> <div className="relative px-2">
{graphRefreshing && renderLoader()} {mainGraphRefreshing && renderLoader()}
<div className="absolute right-4 -top-8 py-1 flex items-center"> <div className="absolute right-4 -top-8 py-1 flex items-center">
{!isRealtime && <StatsExport site={site} query={query} />} {this.downloadLink()}
<SamplingNotice samplePercent={topStatData}/> {this.samplingNotice()}
<WithImportedSwitch site={site} topStatData={topStatData} /> {this.importedNotice()}
<IntervalPicker site={site} query={query} onIntervalUpdate={onIntervalUpdate} /> <IntervalPicker site={site} query={query} graphData={graphData} metric={metric} updateInterval={updateInterval} />
</div> </div>
<LineGraphWithRouter graphData={graphData} darkTheme={isDarkTheme} query={query} /> <FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
</div> </div>
</div>
)
}
}
const LineGraphWithRouter = withRouter(LineGraph)
export default class VisitorGraph extends React.Component {
constructor(props) {
super(props)
this.state = {
topStatsLoadingState: LoadingState.loading,
mainGraphLoadingState: LoadingState.loading,
metric: storage.getItem(`metric__${this.props.site.domain}`) || 'visitors'
}
this.onVisible = this.onVisible.bind(this)
this.updateMetric = this.updateMetric.bind(this)
this.fetchTopStatData = this.fetchTopStatData.bind(this)
this.fetchGraphData = this.fetchGraphData.bind(this)
this.updateInterval = this.updateInterval.bind(this)
}
isIntervalValid(interval) {
const { query, site } = this.props
const validIntervals = site.validIntervalsByPeriod[query.period] || []
return validIntervals.includes(interval)
}
getIntervalFromStorage() {
const { query, site } = this.props
let interval = getStoredInterval(query.period, site.domain)
if (interval !== "week" && interval !== "month" && query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
interval = "month"
}
if (this.isIntervalValid(interval)) {
return interval
} else {
return null
}
}
updateInterval(interval) {
if (this.isIntervalValid(interval)) {
storeInterval(this.props.query.period, this.props.site.domain, interval)
this.setState({ mainGraphLoadingState: LoadingState.refreshing, graphData: null }, this.fetchGraphData)
}
}
onVisible() {
this.setState({ mainGraphLoadingState: LoadingState.loading }, this.fetchGraphData)
this.fetchTopStatData()
if (this.props.query.period === 'realtime') {
document.addEventListener('tick', this.fetchGraphData)
document.addEventListener('tick', this.fetchTopStatData)
}
}
componentDidUpdate(prevProps, prevState) {
const { metric } = this.state;
const { query } = this.props
if (query !== prevProps.query) {
this.setState({ mainGraphLoadingState: LoadingState.loading, topStatsLoadingState: LoadingState.loading, graphData: null, topStatData: null }, this.fetchGraphData)
this.fetchTopStatData()
}
if (metric !== prevState.metric) {
this.setState({ mainGraphLoadingState: LoadingState.refreshing }, this.fetchGraphData)
}
}
resetMetric() {
const { topStatData } = this.state
const { query, site } = this.props
const savedMetric = storage.getItem(`metric__${site.domain}`)
const selectableMetrics = topStatData && topStatData.top_stats.map(({ name }) => METRIC_MAPPING[name]).filter(name => name)
const canSelectSavedMetric = selectableMetrics && selectableMetrics.includes(savedMetric)
if (query.filters.goal) {
this.setState({ metric: 'conversions' })
} else if (canSelectSavedMetric) {
this.setState({ metric: savedMetric })
} else {
this.setState({ metric: 'visitors' })
}
}
componentWillUnmount() {
document.removeEventListener('tick', this.fetchGraphData)
document.removeEventListener('tick', this.fetchTopStatData)
}
storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${this.props.site.domain}`, document.getElementById('top-stats-container').clientHeight)
}
updateMetric(clickedMetric) {
if (this.state.metric == clickedMetric) return
storage.setItem(`metric__${this.props.site.domain}`, clickedMetric)
this.setState({ metric: clickedMetric, graphData: null })
}
fetchGraphData() {
const url = `/api/stats/${encodeURIComponent(this.props.site.domain)}/main-graph`
let params = { metric: this.state.metric }
const interval = this.getIntervalFromStorage()
if (interval) { params.interval = interval }
api.get(url, this.props.query, params)
.then((res) => {
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: res })
return res
})
.catch((err) => {
console.log(err)
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: false })
})
}
fetchTopStatData() {
const query = { ...this.props.query }
if (!isComparisonEnabled(query.comparison)) query.comparison = 'previous_period'
api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/top-stats`, query)
.then((res) => {
this.setState({ topStatsLoadingState: LoadingState.loaded, topStatData: res }, () => {
this.storeTopStatsContainerHeight()
this.resetMetric()
})
return res
})
}
renderInner() {
const { query, site } = this.props;
const { graphData, metric, topStatData, topStatsLoadingState, mainGraphLoadingState } = this.state;
const theme = document.querySelector('html').classList.contains('dark') || false
const mainGraphRefreshing = (mainGraphLoadingState === LoadingState.refreshing)
const topStatAndGraphLoaded = !!(topStatData && graphData)
const shouldShow =
topStatsLoadingState === LoadingState.loaded &&
LoadingState.isLoadedOrRefreshing(mainGraphLoadingState) &&
(topStatData && mainGraphRefreshing || topStatAndGraphLoaded)
return (
<FadeIn show={shouldShow}>
<LineGraphWithRouter mainGraphRefreshing={mainGraphRefreshing} graphData={graphData} topStatData={topStatData} site={site} query={query} darkTheme={theme} metric={metric} updateMetric={this.updateMetric} updateInterval={this.updateInterval} lastLoadTimestamp={this.props.lastLoadTimestamp} />
</FadeIn> </FadeIn>
</div> )
) }
render() {
const { mainGraphLoadingState, topStatsLoadingState } = this.state
const showLoader =
[mainGraphLoadingState, topStatsLoadingState].includes(LoadingState.loading) &&
mainGraphLoadingState !== LoadingState.refreshing
return (
<LazyLoader onVisible={this.onVisible}>
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{showLoader && renderLoader()}
{this.renderInner()}
</div>
</LazyLoader>
)
}
} }
function renderLoader() { function renderLoader() {

@ -1,40 +0,0 @@
import React from "react"
import { parseNaiveDate, isBefore } from '../../util/date'
import { Link } from 'react-router-dom'
import * as url from '../../util/url'
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
export default function WithImportedSwitch({site, topStatData}) {
if (!topStatData?.imports_exist) {
return null
}
function isBeforeNativeStats(date) {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const withImported = topStatData.with_imported;
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
<BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)
} else {
return null
}
}

@ -46,8 +46,7 @@ function PropFilterRow({
<Combobox <Combobox
className="mr-2" className="mr-2"
fetchOptions={fetchPropKeyOptions()} fetchOptions={fetchPropKeyOptions()}
singleOption singleOption={true}
autoFocus
values={propKey ? [propKey] : []} values={propKey ? [propKey] : []}
onSelect={(value) => onPropKeySelect(id, value)} onSelect={(value) => onPropKeySelect(id, value)}
placeholder={'Property'} placeholder={'Property'}

@ -7,7 +7,6 @@ import ListReport from '../reports/list'
import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics'; import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';
import { Menu, Transition } from '@headlessui/react' import { Menu, Transition } from '@headlessui/react'
import { ChevronDownIcon } from '@heroicons/react/20/solid' import { ChevronDownIcon } from '@heroicons/react/20/solid'
import { escapeFilterValue } from '../../util/filters'
import classNames from 'classnames' import classNames from 'classnames'
const UTM_TAGS = { const UTM_TAGS = {
@ -26,7 +25,7 @@ function AllSources(props) {
} }
function getFilterFor(listItem) { function getFilterFor(listItem) {
return { source: escapeFilterValue(listItem['name']) } return { source: listItem['name'] }
} }
function renderIcon(listItem) { function renderIcon(listItem) {
@ -61,7 +60,7 @@ function UTMSources(props) {
} }
function getFilterFor(listItem) { function getFilterFor(listItem) {
return { [props.tab]: escapeFilterValue(listItem['name']) } return { [props.tab]: listItem['name'] }
} }
return ( return (

@ -1,5 +1,3 @@
const flags = JSON.parse(document.getElementById('stats-react-container').dataset.flags)
export const FILTER_GROUPS = { export const FILTER_GROUPS = {
'page': ['page', 'entry_page', 'exit_page'], 'page': ['page', 'entry_page', 'exit_page'],
'source': ['source', 'referrer'], 'source': ['source', 'referrer'],
@ -9,9 +7,7 @@ export const FILTER_GROUPS = {
'os': ['os', 'os_version'], 'os': ['os', 'os_version'],
'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'], 'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'],
'goal': ['goal'], 'goal': ['goal'],
'props': ['prop_key', 'prop_value'], 'props': ['prop_key', 'prop_value']
...(flags.hostname_filter ? { 'hostname': ['hostname'] } : {})
} }
export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location'])) export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location']))
@ -41,7 +37,7 @@ export function isFreeChoiceFilter(filterName) {
let NON_ESCAPED_PIPE_REGEX; let NON_ESCAPED_PIPE_REGEX;
try { try {
NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g") NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g")
} catch (_e) { } catch(_e) {
NON_ESCAPED_PIPE_REGEX = '|' NON_ESCAPED_PIPE_REGEX = '|'
} }
@ -68,19 +64,19 @@ export function parsePrefix(rawValue) {
.filter((clause) => !!clause) .filter((clause) => !!clause)
.map((val) => val.replaceAll(ESCAPED_PIPE, '|')) .map((val) => val.replaceAll(ESCAPED_PIPE, '|'))
return { type, values } return {type, values}
} }
export function parseQueryPropsFilter(query) { export function parseQueryPropsFilter(query) {
return Object.entries(query.filters['props']).map(([key, propVal]) => { return Object.entries(query.filters['props']).map(([key, propVal]) => {
const { type, values } = parsePrefix(propVal) const {type, values} = parsePrefix(propVal)
const clauses = values.map(val => { return { value: val, label: val } }) const clauses = values.map(val => { return {value: val, label: val}})
return { propKey: { label: key, value: key }, type, clauses } return { propKey: { label: key, value: key }, type, clauses }
}) })
} }
export function parseQueryFilter(query, filter) { export function parseQueryFilter(query, filter) {
const { type, values } = parsePrefix(query.filters[filter] || '') const {type, values} = parsePrefix(query.filters[filter] || '')
let labels = values let labels = values
@ -99,9 +95,9 @@ export function parseQueryFilter(query, filter) {
labels = rawLabel.split('|').filter(label => !!label) labels = rawLabel.split('|').filter(label => !!label)
} }
const clauses = values.map((value, index) => { return { value, label: labels[index] } }) const clauses = values.map((value, index) => { return {value, label: labels[index]}})
return { type, clauses } return {type, clauses}
} }
export function isFilteringOnFixedValue(query, filter) { export function isFilteringOnFixedValue(query, filter) {
@ -156,7 +152,6 @@ export const formattedFilters = {
'region': 'Region', 'region': 'Region',
'city': 'City', 'city': 'City',
'page': 'Page', 'page': 'Page',
'hostname': 'Hostname',
'entry_page': 'Entry Page', 'entry_page': 'Entry Page',
'exit_page': 'Exit Page', 'exit_page': 'Exit Page'
} }

@ -19,30 +19,12 @@ if (csrfToken && websocketUrl) {
}) })
} }
} }
let Uploaders = {}
Uploaders.S3 = function (entries, onViewError) {
entries.forEach(entry => {
let xhr = new XMLHttpRequest()
onViewError(() => xhr.abort())
xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
xhr.onerror = () => entry.error()
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
let percent = Math.round((event.loaded / event.total) * 100)
if (percent < 100) { entry.progress(percent) }
}
})
let url = entry.meta.url
xhr.open("PUT", url, true)
xhr.send(entry.file)
})
}
let token = csrfToken.getAttribute("content") let token = csrfToken.getAttribute("content")
let url = websocketUrl.getAttribute("content") let url = websocketUrl.getAttribute("content")
let liveUrl = (url === "") ? "/live" : new URL("/live", url).href; let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
let liveSocket = new LiveSocket(liveUrl, Socket, { let liveSocket = new LiveSocket(liveUrl, Socket, {
heartbeatIntervalMs: 10000, heartbeatIntervalMs: 10000,
params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: { params: { _csrf_token: token }, hooks: Hooks, dom: {
// for alpinejs integration // for alpinejs integration
onBeforeElUpdated(from, to) { onBeforeElUpdated(from, to) {
if (from._x_dataStack) { if (from._x_dataStack) {

@ -25,6 +25,4 @@ S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:10000 S3_ENDPOINT=http://localhost:6000
S3_EXPORTS_BUCKET=dev-exports
S3_IMPORTS_BUCKET=dev-imports

@ -20,6 +20,4 @@ S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:10000 S3_ENDPOINT=http://localhost:6000
S3_EXPORTS_BUCKET=test-exports
S3_IMPORTS_BUCKET=test-imports

@ -38,14 +38,14 @@ config :tailwind,
config :ua_inspector, config :ua_inspector,
database_path: "priv/ua_inspector", database_path: "priv/ua_inspector",
remote_release: "6.3.0" remote_release: "6.2.1"
config :ref_inspector, config :ref_inspector,
database_path: "priv/ref_inspector" database_path: "priv/ref_inspector"
config :plausible, config :plausible,
paddle_api: Plausible.Billing.PaddleApi, paddle_api: Plausible.Billing.PaddleApi,
google_api: Plausible.Google.API google_api: Plausible.Google.Api
config :plausible, config :plausible,
# 30 minutes # 30 minutes
@ -73,8 +73,4 @@ config :ex_cldr,
default_locale: "en", default_locale: "en",
default_backend: Plausible.Cldr default_backend: Plausible.Cldr
config :sentry,
enable_source_code_context: true,
root_source_code_path: [File.cwd!()]
import_config "#{config_env()}.exs" import_config "#{config_env()}.exs"

@ -231,6 +231,10 @@ maxmind_license_key = get_var_from_path_or_env(config_dir, "MAXMIND_LICENSE_KEY"
maxmind_edition = get_var_from_path_or_env(config_dir, "MAXMIND_EDITION", "GeoLite2-City") maxmind_edition = get_var_from_path_or_env(config_dir, "MAXMIND_EDITION", "GeoLite2-City")
persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIR") persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIR")
if System.get_env("DISABLE_AUTH") do
Logger.warning("DISABLE_AUTH env var is no longer supported")
end
enable_email_verification = enable_email_verification =
config_dir config_dir
|> get_var_from_path_or_env("ENABLE_EMAIL_VERIFICATION", "false") |> get_var_from_path_or_env("ENABLE_EMAIL_VERIFICATION", "false")
@ -335,16 +339,21 @@ else
database: get_var_from_path_or_env(config_dir, "DATABASE_NAME", "plausible") database: get_var_from_path_or_env(config_dir, "DATABASE_NAME", "plausible")
end end
included_environments = if sentry_dsn, do: ["prod", "staging", "dev"], else: []
sentry_app_version = runtime_metadata[:version] || app_version sentry_app_version = runtime_metadata[:version] || app_version
config :sentry, config :sentry,
dsn: sentry_dsn, dsn: sentry_dsn,
environment_name: env, environment_name: env,
included_environments: included_environments,
release: sentry_app_version, release: sentry_app_version,
tags: %{app_version: sentry_app_version}, tags: %{app_version: sentry_app_version},
enable_source_code_context: true,
root_source_code_path: [File.cwd!()],
client: Plausible.Sentry.Client, client: Plausible.Sentry.Client,
send_max_attempts: 1, send_max_attempts: 1,
before_send: {Plausible.SentryFilter, :before_send} filter: Plausible.SentryFilter,
before_send_event: {Plausible.SentryFilter, :before_send}
config :plausible, :paddle, config :plausible, :paddle,
vendor_auth_code: paddle_auth_code, vendor_auth_code: paddle_auth_code,
@ -538,9 +547,7 @@ base_queues = [
clean_invitations: 1, clean_invitations: 1,
analytics_imports: 1, analytics_imports: 1,
domain_change_transition: 1, domain_change_transition: 1,
check_accept_traffic_until: 1, check_accept_traffic_until: 1
# NOTE: maybe move s3_csv_export to cloud_queues?
s3_csv_export: 1
] ]
cloud_queues = [ cloud_queues = [
@ -684,6 +691,9 @@ end
config :tzdata, :data_dir, Path.join(persistent_cache_dir || System.tmp_dir!(), "tzdata_data") config :tzdata, :data_dir, Path.join(persistent_cache_dir || System.tmp_dir!(), "tzdata_data")
# Temporarily disable tzdata auto-updating
config :tzdata, :autoupdate, :disabled
promex_disabled? = promex_disabled? =
config_dir config_dir
|> get_var_from_path_or_env("PROMEX_DISABLED", "true") |> get_var_from_path_or_env("PROMEX_DISABLED", "true")
@ -732,14 +742,6 @@ unless s3_disabled? do
%{ %{
name: "S3_ENDPOINT", name: "S3_ENDPOINT",
example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com" example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com"
},
%{
name: "S3_EXPORTS_BUCKET",
example: "my-csv-exports-bucket"
},
%{
name: "S3_IMPORTS_BUCKET",
example: "my-csv-imports-bucket"
} }
] ]
@ -774,8 +776,4 @@ unless s3_disabled? do
scheme: s3_scheme <> "://", scheme: s3_scheme <> "://",
host: s3_host, host: s3_host,
port: s3_port port: s3_port
config :plausible, Plausible.S3,
exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET"),
imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")
end end

@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter
config :plausible, config :plausible,
paddle_api: Plausible.PaddleApi.Mock, paddle_api: Plausible.PaddleApi.Mock,
google_api: Plausible.Google.API.Mock google_api: Plausible.Google.Api.Mock
config :bamboo, :refute_timeout, 10 config :bamboo, :refute_timeout, 10

@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics @revenue_metrics
end end
def total_revenue_query() do def total_revenue_query(query) do
dynamic( from(e in query,
[e], select_merge: %{
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) total_revenue:
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
) )
end end
def average_revenue_query() do def average_revenue_query(query) do
dynamic( from(e in query,
[e], select_merge: %{
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) average_revenue:
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
) )
end end

@ -1,38 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240302"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

@ -1,12 +0,0 @@
{
"account": "accounts/28425555",
"createTime": "2024-02-22T10:50:15.462Z",
"currencyCode": "USD",
"displayName": "account.one - GA4",
"name": "properties/428685444",
"parent": "accounts/28425555",
"propertyType": "PROPERTY_TYPE_ORDINARY",
"serviceLevel": "GOOGLE_ANALYTICS_STANDARD",
"timeZone": "Europe/Warsaw",
"updateTime": "2024-02-22T10:50:15.462Z"
}

@ -1,41 +0,0 @@
{
"accountSummaries": [
{
"account": "accounts/28425178",
"displayName": "account.one",
"name": "accountSummaries/28425178",
"propertySummaries": [
{
"displayName": "account.one - GA4",
"parent": "accounts/28425178",
"property": "properties/428685906",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
},
{
"account": "accounts/45336102",
"displayName": "account.two",
"name": "accountSummaries/45336102"
},
{
"account": "accounts/54516992",
"displayName": "Demo Account",
"name": "accountSummaries/54516992",
"propertySummaries": [
{
"displayName": "GA4 - Flood-It!",
"parent": "accounts/54516992",
"property": "properties/153293282",
"propertyType": "PROPERTY_TYPE_ORDINARY"
},
{
"displayName": "GA4 - Google Merch Shop",
"parent": "accounts/54516992",
"property": "properties/213025502",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,786 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Amsterdam"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 31,
"rows": [
{
"dimensionValues": [
{
"value": "20240131"
}
],
"metricValues": [
{
"value": "141"
},
{
"value": "195"
},
{
"value": "56"
},
{
"value": "146"
},
{
"value": "4815"
}
]
},
{
"dimensionValues": [
{
"value": "20240130"
}
],
"metricValues": [
{
"value": "115"
},
{
"value": "171"
},
{
"value": "42"
},
{
"value": "115"
},
{
"value": "3481"
}
]
},
{
"dimensionValues": [
{
"value": "20240129"
}
],
"metricValues": [
{
"value": "105"
},
{
"value": "153"
},
{
"value": "54"
},
{
"value": "113"
},
{
"value": "2895"
}
]
},
{
"dimensionValues": [
{
"value": "20240128"
}
],
"metricValues": [
{
"value": "184"
},
{
"value": "259"
},
{
"value": "74"
},
{
"value": "190"
},
{
"value": "6861"
}
]
},
{
"dimensionValues": [
{
"value": "20240127"
}
],
"metricValues": [
{
"value": "155"
},
{
"value": "202"
},
{
"value": "67"
},
{
"value": "158"
},
{
"value": "5259"
}
]
},
{
"dimensionValues": [
{
"value": "20240126"
}
],
"metricValues": [
{
"value": "132"
},
{
"value": "189"
},
{
"value": "57"
},
{
"value": "142"
},
{
"value": "5545"
}
]
},
{
"dimensionValues": [
{
"value": "20240125"
}
],
"metricValues": [
{
"value": "151"
},
{
"value": "229"
},
{
"value": "71"
},
{
"value": "155"
},
{
"value": "5365"
}
]
},
{
"dimensionValues": [
{
"value": "20240124"
}
],
"metricValues": [
{
"value": "137"
},
{
"value": "221"
},
{
"value": "64"
},
{
"value": "148"
},
{
"value": "4454"
}
]
},
{
"dimensionValues": [
{
"value": "20240123"
}
],
"metricValues": [
{
"value": "166"
},
{
"value": "267"
},
{
"value": "61"
},
{
"value": "171"
},
{
"value": "4850"
}
]
},
{
"dimensionValues": [
{
"value": "20240122"
}
],
"metricValues": [
{
"value": "161"
},
{
"value": "233"
},
{
"value": "73"
},
{
"value": "165"
},
{
"value": "4871"
}
]
},
{
"dimensionValues": [
{
"value": "20240121"
}
],
"metricValues": [
{
"value": "254"
},
{
"value": "397"
},
{
"value": "114"
},
{
"value": "267"
},
{
"value": "8845"
}
]
},
{
"dimensionValues": [
{
"value": "20240120"
}
],
"metricValues": [
{
"value": "214"
},
{
"value": "383"
},
{
"value": "100"
},
{
"value": "223"
},
{
"value": "6914"
}
]
},
{
"dimensionValues": [
{
"value": "20240119"
}
],
"metricValues": [
{
"value": "381"
},
{
"value": "5571"
},
{
"value": "100"
},
{
"value": "228"
},
{
"value": "98549"
}
]
},
{
"dimensionValues": [
{
"value": "20240118"
}
],
"metricValues": [
{
"value": "177"
},
{
"value": "278"
},
{
"value": "86"
},
{
"value": "186"
},
{
"value": "5553"
}
]
},
{
"dimensionValues": [
{
"value": "20240117"
}
],
"metricValues": [
{
"value": "204"
},
{
"value": "271"
},
{
"value": "97"
},
{
"value": "216"
},
{
"value": "6777"
}
]
},
{
"dimensionValues": [
{
"value": "20240116"
}
],
"metricValues": [
{
"value": "200"
},
{
"value": "280"
},
{
"value": "65"
},
{
"value": "203"
},
{
"value": "9381"
}
]
},
{
"dimensionValues": [
{
"value": "20240115"
}
],
"metricValues": [
{
"value": "255"
},
{
"value": "311"
},
{
"value": "86"
},
{
"value": "269"
},
{
"value": "12365"
}
]
},
{
"dimensionValues": [
{
"value": "20240114"
}
],
"metricValues": [
{
"value": "243"
},
{
"value": "302"
},
{
"value": "86"
},
{
"value": "257"
},
{
"value": "11073"
}
]
},
{
"dimensionValues": [
{
"value": "20240113"
}
],
"metricValues": [
{
"value": "231"
},
{
"value": "276"
},
{
"value": "92"
},
{
"value": "244"
},
{
"value": "10084"
}
]
},
{
"dimensionValues": [
{
"value": "20240112"
}
],
"metricValues": [
{
"value": "215"
},
{
"value": "265"
},
{
"value": "75"
},
{
"value": "225"
},
{
"value": "10195"
}
]
},
{
"dimensionValues": [
{
"value": "20240111"
}
],
"metricValues": [
{
"value": "183"
},
{
"value": "235"
},
{
"value": "64"
},
{
"value": "191"
},
{
"value": "8200"
}
]
},
{
"dimensionValues": [
{
"value": "20240110"
}
],
"metricValues": [
{
"value": "190"
},
{
"value": "244"
},
{
"value": "61"
},
{
"value": "203"
},
{
"value": "9056"
}
]
},
{
"dimensionValues": [
{
"value": "20240109"
}
],
"metricValues": [
{
"value": "201"
},
{
"value": "290"
},
{
"value": "63"
},
{
"value": "213"
},
{
"value": "11651"
}
]
},
{
"dimensionValues": [
{
"value": "20240108"
}
],
"metricValues": [
{
"value": "195"
},
{
"value": "235"
},
{
"value": "71"
},
{
"value": "209"
},
{
"value": "7946"
}
]
},
{
"dimensionValues": [
{
"value": "20240107"
}
],
"metricValues": [
{
"value": "205"
},
{
"value": "274"
},
{
"value": "80"
},
{
"value": "218"
},
{
"value": "10048"
}
]
},
{
"dimensionValues": [
{
"value": "20240106"
}
],
"metricValues": [
{
"value": "256"
},
{
"value": "303"
},
{
"value": "119"
},
{
"value": "281"
},
{
"value": "8440"
}
]
},
{
"dimensionValues": [
{
"value": "20240105"
}
],
"metricValues": [
{
"value": "226"
},
{
"value": "270"
},
{
"value": "83"
},
{
"value": "237"
},
{
"value": "9311"
}
]
},
{
"dimensionValues": [
{
"value": "20240104"
}
],
"metricValues": [
{
"value": "223"
},
{
"value": "274"
},
{
"value": "81"
},
{
"value": "236"
},
{
"value": "12051"
}
]
},
{
"dimensionValues": [
{
"value": "20240103"
}
],
"metricValues": [
{
"value": "233"
},
{
"value": "283"
},
{
"value": "79"
},
{
"value": "245"
},
{
"value": "10628"
}
]
},
{
"dimensionValues": [
{
"value": "20240102"
}
],
"metricValues": [
{
"value": "203"
},
{
"value": "269"
},
{
"value": "60"
},
{
"value": "210"
},
{
"value": "11168"
}
]
},
{
"dimensionValues": [
{
"value": "20240101"
}
],
"metricValues": [
{
"value": "191"
},
{
"value": "224"
},
{
"value": "71"
},
{
"value": "197"
},
{
"value": "8080"
}
]
}
]
}
]
}

@ -1,38 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240222"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20200421"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20120118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20170118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -24,7 +24,6 @@ defmodule Mix.Tasks.SendPageview do
page: :string, page: :string,
referrer: :string, referrer: :string,
host: :string, host: :string,
hostname: :string,
event: :string, event: :string,
props: :string, props: :string,
revenue_currency: :string, revenue_currency: :string,
@ -87,7 +86,6 @@ defmodule Mix.Tasks.SendPageview do
referrer = Keyword.get(opts, :referrer, @default_referrer) referrer = Keyword.get(opts, :referrer, @default_referrer)
event = Keyword.get(opts, :event, @default_event) event = Keyword.get(opts, :event, @default_event)
props = Keyword.get(opts, :props, @default_props) props = Keyword.get(opts, :props, @default_props)
hostname = Keyword.get(opts, :hostname, domain)
revenue = revenue =
if Keyword.get(opts, :revenue_currency) do if Keyword.get(opts, :revenue_currency) do
@ -99,7 +97,7 @@ defmodule Mix.Tasks.SendPageview do
%{ %{
name: event, name: event,
url: "http://#{hostname}#{page}", url: "http://#{domain}#{page}",
domain: domain, domain: domain,
referrer: referrer, referrer: referrer,
props: props, props: props,

@ -10,7 +10,6 @@ defmodule Plausible.Application do
on_full_build(do: Plausible.License.ensure_valid_license()) on_full_build(do: Plausible.License.ensure_valid_license())
children = [ children = [
Plausible.Cache.Stats,
Plausible.Repo, Plausible.Repo,
Plausible.ClickhouseRepo, Plausible.ClickhouseRepo,
Plausible.IngestRepo, Plausible.IngestRepo,
@ -24,15 +23,13 @@ defmodule Plausible.Application do
Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),
ReferrerBlocklist, ReferrerBlocklist,
Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents, Supervisor.child_spec({Cachex, name: :user_agents, limit: 10_000, stats: true},
ttl_check_interval: :timer.seconds(5), id: :cachex_user_agents
global_ttl: :timer.minutes(60)
), ),
Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions, Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},
ttl_check_interval: :timer.seconds(1), id: :cachex_sessions
global_ttl: :timer.minutes(30)
), ),
{Plausible.Site.Cache, ttl_check_interval: false}, {Plausible.Site.Cache, []},
{Plausible.Cache.Warmer, {Plausible.Cache.Warmer,
[ [
child_name: Plausible.Site.Cache.All, child_name: Plausible.Site.Cache.All,
@ -47,7 +44,7 @@ defmodule Plausible.Application do
interval: :timer.seconds(30), interval: :timer.seconds(30),
warmer_fn: :refresh_updated_recently warmer_fn: :refresh_updated_recently
]}, ]},
{Plausible.Shield.IPRuleCache, ttl_check_interval: false}, {Plausible.Shield.IPRuleCache, []},
{Plausible.Cache.Warmer, {Plausible.Cache.Warmer,
[ [
child_name: Plausible.Shield.IPRuleCache.All, child_name: Plausible.Shield.IPRuleCache.All,
@ -62,7 +59,7 @@ defmodule Plausible.Application do
interval: :timer.seconds(35), interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently warmer_fn: :refresh_updated_recently
]}, ]},
{Plausible.Shield.CountryRuleCache, ttl_check_interval: false}, {Plausible.Shield.CountryRuleCache, []},
{Plausible.Cache.Warmer, {Plausible.Cache.Warmer,
[ [
child_name: Plausible.Shield.CountryRuleCache.All, child_name: Plausible.Shield.CountryRuleCache.All,
@ -77,21 +74,6 @@ defmodule Plausible.Application do
interval: :timer.seconds(35), interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently warmer_fn: :refresh_updated_recently
]}, ]},
{Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.All,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Auth.TOTP.Vault, key: totp_vault_key()}, {Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
PlausibleWeb.Endpoint, PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)}, {Oban, Application.get_env(:plausible, Oban)},
@ -137,10 +119,10 @@ defmodule Plausible.Application do
defp maybe_add_sentry_pool(pool_config) do defp maybe_add_sentry_pool(pool_config) do
case Sentry.Config.dsn() do case Sentry.Config.dsn() do
{"http" <> _rest = url, _, _} -> dsn when is_binary(dsn) ->
Map.put(pool_config, url, size: 50) Map.put(pool_config, dsn, size: 50)
nil -> _ ->
pool_config pool_config
end end
end end
@ -186,6 +168,16 @@ defmodule Plausible.Application do
) )
end end
def report_cache_stats() do
case Cachex.stats(:user_agents) do
{:ok, stats} ->
Logger.info("User agent cache stats: #{inspect(stats)}")
e ->
IO.puts("Unable to show cache stats: #{inspect(e)}")
end
end
defp setup_opentelemetry() do defp setup_opentelemetry() do
OpentelemetryPhoenix.setup() OpentelemetryPhoenix.setup()
OpentelemetryEcto.setup([:plausible, :repo]) OpentelemetryEcto.setup([:plausible, :repo])

@ -8,10 +8,10 @@ defmodule Plausible.Cache do
# - Optionally override `unwrap_cache_keys/1` # - Optionally override `unwrap_cache_keys/1`
# - Populate the cache with `Plausible.Cache.Warmer` # - Populate the cache with `Plausible.Cache.Warmer`
Serves as a wrapper around `Plausible.Cache.Adapter`, where the underlying Serves as a thin wrapper around Cachex, but the underlying
implementation can be transparently swapped. implementation can be transparently swapped.
Even though normally the relevant Adapter processes are started, cache access is disabled Even though the Cachex process is started, cache access is disabled
during tests via the `:plausible, #{__MODULE__}, enabled: bool()` application env key. during tests via the `:plausible, #{__MODULE__}, enabled: bool()` application env key.
This can be overridden on case by case basis, using the child specs options. This can be overridden on case by case basis, using the child specs options.
@ -61,30 +61,26 @@ defmodule Plausible.Cache do
@behaviour Plausible.Cache @behaviour Plausible.Cache
@modes [:all, :updated_recently] @modes [:all, :updated_recently]
alias Plausible.Cache.Adapter
@spec get(any(), Keyword.t()) :: any() | nil @spec get(any(), Keyword.t()) :: any() | nil
def get(key, opts \\ []) when is_list(opts) do def get(key, opts \\ []) do
cache_name = Keyword.get(opts, :cache_name, name()) cache_name = Keyword.get(opts, :cache_name, name())
force? = Keyword.get(opts, :force?, false) force? = Keyword.get(opts, :force?, false)
if Plausible.Cache.enabled?() or force? do if Plausible.Cache.enabled?() or force? do
Adapter.get(cache_name, key) case Cachex.get(cache_name, key) do
else {:ok, nil} ->
get_from_source(key) nil
end
end
@spec get_or_store(any(), (-> any()), Keyword.t()) :: any() | nil {:ok, item} ->
def get_or_store(key, fallback_fn, opts \\ []) item
when is_function(fallback_fn, 0) and is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name())
force? = Keyword.get(opts, :force?, false)
if Plausible.Cache.enabled?() or force? do {:error, e} ->
Adapter.get(cache_name, key, fallback_fn) Logger.error("Error retrieving key from '#{inspect(cache_name)}': #{inspect(e)}")
nil
end
else else
get_from_source(key) || fallback_fn.() get_from_source(key)
end end
end end
@ -121,10 +117,10 @@ defmodule Plausible.Cache do
def merge_items(new_items, opts) do def merge_items(new_items, opts) do
new_items = unwrap_cache_keys(new_items) new_items = unwrap_cache_keys(new_items)
cache_name = Keyword.get(opts, :cache_name, name()) cache_name = Keyword.get(opts, :cache_name, name())
:ok = Adapter.put_many(cache_name, new_items) true = Cachex.put_many!(cache_name, new_items)
if Keyword.get(opts, :delete_stale_items?, true) do if Keyword.get(opts, :delete_stale_items?, true) do
old_keys = Adapter.keys(cache_name) {:ok, old_keys} = Cachex.keys(cache_name)
new = MapSet.new(Enum.into(new_items, [], fn {k, _} -> k end)) new = MapSet.new(Enum.into(new_items, [], fn {k, _} -> k end))
old = MapSet.new(old_keys) old = MapSet.new(old_keys)
@ -132,7 +128,7 @@ defmodule Plausible.Cache do
old old
|> MapSet.difference(new) |> MapSet.difference(new)
|> Enum.each(fn k -> |> Enum.each(fn k ->
Adapter.delete(cache_name, k) Cachex.del(cache_name, k)
end) end)
end end
@ -143,7 +139,11 @@ defmodule Plausible.Cache do
def child_spec(opts) do def child_spec(opts) do
cache_name = Keyword.get(opts, :cache_name, name()) cache_name = Keyword.get(opts, :cache_name, name())
child_id = Keyword.get(opts, :child_id, child_id()) child_id = Keyword.get(opts, :child_id, child_id())
Adapter.child_spec(cache_name, child_id, opts)
Supervisor.child_spec(
{Cachex, name: cache_name, limit: nil, stats: true},
id: child_id
)
end end
@doc """ @doc """
@ -154,7 +154,7 @@ defmodule Plausible.Cache do
@spec ready?(atom()) :: boolean @spec ready?(atom()) :: boolean
def ready?(cache_name \\ name()) do def ready?(cache_name \\ name()) do
case size(cache_name) do case size(cache_name) do
n when is_integer(n) and n > 0 -> n when n > 0 ->
true true
0 -> 0 ->
@ -165,8 +165,21 @@ defmodule Plausible.Cache do
end end
end end
defdelegate size(cache_name \\ name()), to: Plausible.Cache.Adapter @spec size() :: non_neg_integer()
defdelegate hit_rate(cache_name \\ name()), to: Plausible.Cache.Stats def size(cache_name \\ name()) do
case Cachex.size(cache_name) do
{:ok, size} -> size
_ -> 0
end
end
@spec hit_rate() :: number()
def hit_rate(cache_name \\ name()) do
case Cachex.stats(cache_name) do
{:ok, stats} -> Map.get(stats, :hit_rate, 0)
_ -> 0
end
end
@spec telemetry_event_refresh(atom(), atom()) :: list(atom()) @spec telemetry_event_refresh(atom(), atom()) :: list(atom())
def telemetry_event_refresh(cache_name \\ name(), mode) when mode in @modes do def telemetry_event_refresh(cache_name \\ name(), mode) when mode in @modes do
@ -176,16 +189,26 @@ defmodule Plausible.Cache do
defp refresh(mode, query, opts) when mode in @modes do defp refresh(mode, query, opts) when mode in @modes do
cache_name = Keyword.get(opts, :cache_name, name()) cache_name = Keyword.get(opts, :cache_name, name())
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( measure_duration(telemetry_event_refresh(cache_name, mode), fn ->
telemetry_event_refresh(cache_name, mode), items = Plausible.Repo.all(query)
fn -> :ok = merge_items(items, opts)
items = Plausible.Repo.all(query) end)
:ok = merge_items(items, opts)
end
)
:ok :ok
end end
defp measure_duration(event, fun) when is_function(fun, 0) do
{duration, result} = time_it(fun)
:telemetry.execute(event, %{duration: duration}, %{})
result
end
defp time_it(fun) do
start = System.monotonic_time()
result = fun.()
stop = System.monotonic_time()
{stop - start, result}
end
end end
end end
end end

@ -1,136 +0,0 @@
defmodule Plausible.Cache.Adapter do
@moduledoc """
Interface for the underlying cache implementation.
Currently: ConCache
Using the Adapter module directly, the user must ensure that the relevant
processes are available to use, which is normally done via the child specification.
"""
require Logger
@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()
def child_spec(name, child_id, opts \\ [])
when is_atom(name) and is_atom(child_id) and is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name)
child_id = Keyword.get(opts, :child_id, child_id)
ttl_check_interval = Keyword.get(opts, :ttl_check_interval, false)
opts =
opts
|> Keyword.put(:name, cache_name)
|> Keyword.put(:ttl_check_interval, ttl_check_interval)
Supervisor.child_spec(
{ConCache, opts},
id: child_id
)
end
@spec size(atom()) :: non_neg_integer() | nil
def size(cache_name) do
ConCache.size(cache_name)
catch
:exit, _ -> nil
end
@spec get(atom(), any()) :: any()
def get(cache_name, key) do
result = ConCache.get(cache_name, key)
if is_nil(result), do: miss(cache_name), else: hit(cache_name)
result
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
nil
end
@spec get(atom(), any(), (-> any())) :: any()
def get(cache_name, key, fallback_fn) do
cache = ConCache.Owner.cache(cache_name)
case ConCache.Operations.get(cache, key) do
nil ->
get_or_store_isolated(cache, cache_name, key, fallback_fn)
value ->
hit(cache_name)
value
end
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
nil
end
@spec put(atom(), any(), any()) :: any()
def put(cache_name, key, value) do
:ok = ConCache.put(cache_name, key, value)
value
catch
:exit, _ ->
Logger.error("Error putting a key to '#{cache_name}'")
nil
end
@spec put_many(atom(), [any()]) :: :ok
def put_many(cache_name, items) when is_list(items) do
true = :ets.insert(ConCache.ets(cache_name), items)
:ok
catch
:exit, _ ->
Logger.error("Error putting keys to '#{cache_name}'")
:ok
end
@spec delete(atom(), any()) :: :ok
def delete(cache_name, key) do
ConCache.dirty_delete(cache_name, key)
catch
:exit, _ ->
Logger.error("Error deleting a key in '#{cache_name}'")
:ok
end
@spec keys(atom()) :: Enumerable.t()
def keys(cache_name) do
ets = ConCache.ets(cache_name)
Stream.resource(
fn -> :ets.first(ets) end,
fn
:"$end_of_table" -> {:halt, nil}
prev_key -> {[prev_key], :ets.next(ets, prev_key)}
end,
fn _ -> :ok end
)
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
[]
end
defp hit(cache_name) do
Plausible.Cache.Stats.record_hit(cache_name)
end
defp miss(cache_name) do
Plausible.Cache.Stats.record_miss(cache_name)
end
defp get_or_store_isolated(cache, cache_name, key, fun) do
ConCache.isolated(cache_name, key, fn ->
case ConCache.Operations.get(cache, key) do
nil ->
new_value = fun.()
ConCache.Operations.dirty_put(cache, key, new_value)
miss(cache_name)
new_value
existing ->
hit(cache_name)
existing
end
end)
end
end

@ -1,84 +0,0 @@
defmodule Plausible.Cache.Stats do
@moduledoc """
Keeps track of hit/miss ratio for various caches.
"""
use GenServer
@hit :hit
@miss :miss
@telemetry_hit [:plausible, :cache, :adapter, @hit]
@telemetry_miss [:plausible, :cache, :adapter, @miss]
@telemetry_events [@telemetry_hit, @telemetry_miss]
def start_link(_opts) do
GenServer.start_link(__MODULE__, nil)
end
def record_hit(cache_name) do
:telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name})
end
def record_miss(cache_name) do
:telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name})
end
def init(nil) do
__MODULE__ =
:ets.new(__MODULE__, [
:public,
:named_table,
:set,
read_concurrency: true,
write_concurrency: true
])
:telemetry.attach_many(
"plausible-cache-stats",
@telemetry_events,
&__MODULE__.handle_telemetry_event/4,
nil
)
{:ok, nil}
end
def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do
bump(cache_name, @hit)
end
def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do
bump(cache_name, @miss)
end
def gather(cache_name) do
{:ok,
%{
hit_rate: hit_rate(cache_name),
count: size(cache_name) || 0
}}
end
defdelegate size(cache_name), to: Plausible.Cache.Adapter
def bump(cache_name, type) do
:ets.update_counter(
__MODULE__,
{cache_name, type},
1,
{{cache_name, type}, 0}
)
end
def hit_rate(cache_name) do
hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)
miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)
hit_miss = hit + miss
if hit_miss == 0 do
0.0
else
hit / hit_miss * 100
end
end
end

@ -65,28 +65,4 @@ defmodule Plausible.ClickhouseEventV2 do
) )
|> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp]) |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
end end
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
def merge_session(%__MODULE__{} = event, session) do
Map.merge(event, Map.take(session, @session_properties))
end
end end

@ -43,7 +43,6 @@ defmodule Plausible.ClickhouseSessionV2 do
field :is_bounce, BoolUInt8 field :is_bounce, BoolUInt8
field :entry_page, :string field :entry_page, :string
field :exit_page, :string field :exit_page, :string
field :exit_page_hostname, :string
field :pageviews, Ch, type: "Int32" field :pageviews, Ch, type: "Int32"
field :events, Ch, type: "Int32" field :events, Ch, type: "Int32"
field :sign, Ch, type: "Int8" field :sign, Ch, type: "Int8"

@ -1,188 +0,0 @@
defmodule Plausible.DataMigration.SiteImports do
@moduledoc """
Site imports migration backfilling SiteImport entries for old imports
and alters import end dates to match actual end date of respective import stats.
"""
import Ecto.Query
alias Plausible.ClickhouseRepo
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
def run(opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
sites_with_imports =
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|> Repo.all(log: false)
sites_count = length(sites_with_imports)
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
for {site, idx} <- Enum.with_index(sites_with_imports) do
site_imports =
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|> Repo.all(log: false)
IO.puts(
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
)
site_imports =
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
IO.puts("Creating legacy site import entry for site ID #{site.id}")
# create legacy entry if there's not one yet
params =
site.imported_data
|> Imported.SiteImport.from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
legacy_site_import =
%Imported.SiteImport{}
|> Ecto.Changeset.change(params)
|> insert!(dry_run?)
[legacy_site_import | site_imports]
else
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
site_imports
end
# adjust end date for each site import
for site_import <- site_imports do
IO.puts(
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
)
import_ids =
if site_import.legacy do
[0, site_import.id]
else
[site_import.id]
end
end_date = imported_stats_end_date(site.id, import_ids)
if !end_date do
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
)
if site_import.legacy do
clear_imported_data(site, dry_run?)
end
delete!(site_import, dry_run?)
else
end_date =
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
end_date
else
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
)
site_import.end_date
end
site_import
|> Ecto.Changeset.change(end_date: end_date)
|> update!(dry_run?)
IO.puts(
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
)
end
end
IO.puts("Done processing site ID #{site.id}")
end
IO.puts("Finished")
end
# Exposed for testing purposes
@doc false
def imported_stats_end_date(site_id, import_ids) do
[first_schema | schemas] = Imported.schemas()
query =
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
end)
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
if dates != [] do
case Enum.max(dates, Date) do
# no stats for this domain yet
~D[1970-01-01] ->
nil
date ->
date
end
else
nil
end
end
defp insert!(changeset, false = _dry_run?) do
Repo.insert!(changeset)
end
defp insert!(changeset, true = _dry_run?) do
if changeset.valid? do
changeset
|> Ecto.Changeset.change(id: 0)
|> Ecto.Changeset.apply_changes()
else
raise "Invalid insert: #{inspect(changeset)}"
end
end
defp clear_imported_data(site, false = _dry_run?) do
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
end
defp clear_imported_data(site, true = _dry_run?) do
%{site | imported_data: nil}
end
defp update!(changeset, false = _dry_run?) do
Repo.update!(changeset)
end
defp update!(changeset, true = _dry_run?) do
if changeset.valid? do
Ecto.Changeset.apply_changes(changeset)
else
raise "Invalid update: #{inspect(changeset)}"
end
end
defp delete!(entity, false = _dry_run?) do
Repo.delete!(entity)
end
defp delete!(entity, true = _dry_run?) do
entity
end
defp max_date_query(schema, site_id, import_ids) do
from(q in schema,
where: q.site_id == ^site_id,
where: q.import_id in ^import_ids,
select: %{max_date: fragment("max(?)", q.date)}
)
end
end

@ -1,371 +0,0 @@
defmodule Plausible.Exports do
@moduledoc """
Contains functions to export data for events and sessions as Zip archives.
"""
require Plausible
import Ecto.Query
@doc """
Renders filename for the Zip archive containing the exported CSV files.
Examples:
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
"plausible_io_20210101_20241231.zip"
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
"Bücher_example_20210101_20241231.zip"
"""
def archive_filename(domain, min_date, max_date) do
name =
Enum.join(
[
String.replace(domain, ".", "_"),
Calendar.strftime(min_date, "%Y%m%d"),
Calendar.strftime(max_date, "%Y%m%d")
],
"_"
)
name <> ".zip"
end
@doc """
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
tables into the format of `imported_*` tables for a website.
"""
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
%{String.t() => Ecto.Query.t()}
def export_queries(site_id, opts \\ []) do
extname = opts[:extname] || ".csv"
date_range = opts[:date_range]
filename = fn table ->
name =
if date_range do
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
"#{table}_#{first_date}_#{last_date}"
else
table
end
name <> extname
end
%{
filename.("imported_visitors") => export_visitors_q(site_id),
filename.("imported_sources") => export_sources_q(site_id),
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
filename.("imported_pages") => export_pages_q(site_id),
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
filename.("imported_locations") => export_locations_q(site_id),
filename.("imported_devices") => export_devices_q(site_id),
filename.("imported_browsers") => export_browsers_q(site_id),
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
}
end
Plausible.on_full_build do
defp sampled(table) do
Plausible.Stats.Sampling.add_query_hint(from(table))
end
else
defp sampled(table) do
table
end
end
defmacrop date(timestamp) do
quote do
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
end
end
defmacrop visit_duration(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
:visit_duration
)
end
end
defmacrop visitors(t) do
quote do
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
:visitors
)
end
end
defmacrop visits(t) do
quote do
selected_as(sum(unquote(t).sign), :visits)
end
end
defmacrop bounces(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
:bounces
)
end
end
defmacrop pageviews(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
:pageviews
)
end
end
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
def export_visitors_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: selected_as(:date),
select: [
date(s.start),
visitors(s),
pageviews(s),
bounces(s),
visits(s),
visit_duration(s)
]
end
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
def export_sources_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [
selected_as(:date),
selected_as(:source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
s.utm_term
],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.referrer_source, :source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
s.utm_term,
pageviews(s),
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
]
end
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
def export_pages_q(site_id) do
window_q =
from e in sampled("events_v2"),
where: e.site_id == ^site_id,
select: %{
timestamp: e.timestamp,
next_timestamp:
over(fragment("leadInFrame(?)", e.timestamp),
partition_by: e.session_id,
order_by: e.timestamp,
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
),
pathname: e.pathname,
hostname: e.hostname,
name: e.name,
user_id: e.user_id,
session_id: e.session_id,
_sample_factor: fragment("_sample_factor")
}
from e in subquery(window_q),
group_by: [selected_as(:date), e.pathname],
order_by: selected_as(:date),
select: [
date(e.timestamp),
selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page),
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
:visits
),
visitors(e),
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
:pageviews
),
# NOTE: are exits pageviews or any events?
selected_as(
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
:exits
),
selected_as(
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
:time_on_page
)
]
end
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
def export_entry_pages_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.entry_page],
order_by: selected_as(:date),
select: [
date(s.start),
s.entry_page,
visitors(s),
selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:entrances
),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
def export_exit_pages_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.exit_page],
order_by: selected_as(:date),
select: [
date(s.start),
s.exit_page,
visitors(s),
visit_duration(s),
selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:exits
),
bounces(s),
pageviews(s)
]
end
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
def export_locations_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.country_code, :country),
selected_as(s.subdivision1_code, :region),
selected_as(s.city_geoname_id, :city),
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
def export_devices_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.screen_size],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.screen_size, :device),
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
def export_browsers_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.browser, s.browser_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.browser,
s.browser_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
def export_operating_systems_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.operating_system,
s.operating_system_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@doc """
Creates a streamable Zip archive from the provided (named) Ecto queries.
Example usage:
{:ok, pool} = Ch.start_link(pool_size: 1)
DBConnection.run(pool, fn conn ->
conn
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|> Stream.into(File.stream!("export.zip"))
|> Stream.run()
end)
"""
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
Enumerable.t()
def stream_archive(conn, named_queries, opts \\ []) do
entries =
Enum.map(named_queries, fn {name, query} ->
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
datastream =
conn
|> Ch.stream(sql, params, opts)
|> Stream.map(fn %Ch.Result{data: data} -> data end)
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
end)
Zstream.zip(entries)
end
end

@ -80,14 +80,6 @@ defmodule Plausible.Goals do
def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"} def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"}
def list_revenue_goals(site) do
from(g in Plausible.Goal,
where: g.site_id == ^site.id and not is_nil(g.currency),
select: %{event_name: g.event_name, currency: g.currency}
)
|> Plausible.Repo.all()
end
def for_site(site, opts \\ []) do def for_site(site, opts \\ []) do
site site
|> for_site_query(opts) |> for_site_query(opts)

@ -1,14 +1,10 @@
defmodule Plausible.Google.API do defmodule Plausible.Google.Api do
@moduledoc """ alias Plausible.Google.{ReportRequest, HTTP}
API to Google services.
"""
use Timex use Timex
alias Plausible.Google.HTTP
require Logger require Logger
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
@search_console_scope URI.encode_www_form( @search_console_scope URI.encode_www_form(
"email https://www.googleapis.com/auth/webmasters.readonly" "email https://www.googleapis.com/auth/webmasters.readonly"
) )
@ -21,48 +17,11 @@ defmodule Plausible.Google.API do
Jason.encode!([site_id, redirect_to]) Jason.encode!([site_id, redirect_to])
end end
def import_authorize_url(site_id, redirect_to, opts \\ []) do def import_authorize_url(site_id, redirect_to, legacy \\ true) do
legacy = Keyword.get(opts, :legacy, true)
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
Jason.encode!([site_id, redirect_to, legacy]) Jason.encode!([site_id, redirect_to, legacy])
end end
def fetch_access_token!(code) do
HTTP.fetch_access_token!(code)
end
def list_properties_and_views(access_token) do
with {:ok, properties} <- Plausible.Google.GA4.API.list_properties(access_token),
{:ok, views} <- Plausible.Google.UA.API.list_views(access_token) do
{:ok, properties ++ views}
end
end
def get_property_or_view(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_property(access_token, property_or_view)
else
Plausible.Google.UA.API.get_view(access_token, property_or_view)
end
end
def get_analytics_start_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_start_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_start_date(access_token, property_or_view)
end
end
def get_analytics_end_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view)
end
end
def fetch_verified_properties(auth) do def fetch_verified_properties(auth) do
with {:ok, access_token} <- maybe_refresh_token(auth), with {:ok, access_token} <- maybe_refresh_token(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do
@ -94,7 +53,138 @@ defmodule Plausible.Google.API do
end end
end end
def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do @spec list_views(access_token :: String.t()) ::
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
@doc """
Lists Google Analytics views grouped by hostname.
"""
def list_views(access_token) do
case HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
views = Enum.group_by(views, &view_hostname/1, &view_names/1)
{:ok, views}
error ->
error
end
end
defp view_hostname(view) do
case view do
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
_any -> "Others"
end
end
defp view_names(%{"name" => name, "id" => id}) do
{"#{id} - #{name}", id}
end
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
@doc """
Returns a single Google Analytics view if the user has access to it.
"""
def get_view(access_token, lookup_id) do
case list_views(access_token) do
{:ok, views} ->
view =
views
|> Map.values()
|> List.flatten()
|> Enum.find(fn {_name, id} -> id == lookup_id end)
{:ok, view}
{:error, cause} ->
{:error, cause}
end
end
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 7_500
@backoff_factor :timer.seconds(10)
@max_attempts 5
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
This function fetches Google Analytics reports in batches of #{@per_page} per
request. The batches are then passed to persist callback.
Requests to Google Analytics can fail, and are retried at most
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
importing has finished or `{:error, term()}` when a request to GA failed too
many times.
Useful links:
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(date_range, view_id, auth, persist_fn) do
with {:ok, access_token} <- maybe_refresh_token(auth) do
do_import_analytics(date_range, view_id, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok ->
report_request = %ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case HTTP.get_report(report_request) do
{:ok, {rows, next_page_token}} ->
:ok = persist_fn.(report_request.dataset, rows)
if next_page_token do
fetch_and_persist(
%ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
{:error, cause}
else
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
with true <- needs_to_refresh_token?(auth.expires), with true <- needs_to_refresh_token?(auth.expires),
{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
changeset <- changeset <-
@ -110,7 +200,7 @@ defmodule Plausible.Google.API do
end end
end end
def maybe_refresh_token({access_token, refresh_token, expires_at}) do defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
with true <- needs_to_refresh_token?(expires_at), with true <- needs_to_refresh_token?(expires_at),
{:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do
{:ok, new_access_token} {:ok, new_access_token}
@ -120,8 +210,6 @@ defmodule Plausible.Google.API do
end end
end end
def property?(value), do: String.starts_with?(value, "properties/")
defp do_refresh_token(refresh_token) do defp do_refresh_token(refresh_token) do
case HTTP.refresh_auth_token(refresh_token) do case HTTP.refresh_auth_token(refresh_token) do
{:ok, %{"access_token" => new_access_token, "expires_in" => expires_in}} -> {:ok, %{"access_token" => new_access_token, "expires_in" => expires_in}} ->

@ -1,152 +0,0 @@
defmodule Plausible.Google.GA4.API do
@moduledoc """
API for Google Analytics 4.
"""
alias Plausible.Google
alias Plausible.Google.GA4
require Logger
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 50_000
@backoff_factor :timer.seconds(10)
@max_attempts 5
def list_properties(access_token) do
case GA4.HTTP.list_accounts_for_user(access_token) do
{:ok, %{"accountSummaries" => accounts}} ->
accounts =
accounts
|> Enum.filter(& &1["propertySummaries"])
|> Enum.map(fn account ->
%{"account" => account_id, "displayName" => account_name} = account
{"#{account_name} (#{account_id})",
Enum.map(account["propertySummaries"], fn property ->
%{"displayName" => property_name, "property" => property_id} = property
{"#{property_name} (#{property_id})", property_id}
end)}
end)
{:ok, accounts}
{:error, cause} ->
{:error, cause}
end
end
def get_property(access_token, lookup_property) do
case GA4.HTTP.get_property(access_token, lookup_property) do
{:ok, property} ->
%{"displayName" => property_name, "name" => property_id, "account" => account_id} =
property
{:ok,
%{
id: property_id,
name: "#{property_name} (#{property_id})",
account_id: account_id
}}
{:error, cause} ->
{:error, cause}
end
end
def get_analytics_start_date(access_token, property) do
GA4.HTTP.get_analytics_start_date(access_token, property)
end
def get_analytics_end_date(access_token, property) do
GA4.HTTP.get_analytics_end_date(access_token, property)
end
def import_analytics(date_range, property, auth, persist_fn) do
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
)
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
do_import_analytics(date_range, property, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, property, access_token, persist_fn) do
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
)
report_request = prepare_request(report_request, date_range, property, access_token)
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case GA4.HTTP.get_report(report_request) do
{:ok, {rows, row_count}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
)
:ok = persist_fn.(report_request.dataset, rows)
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
)
if report_request.offset + @per_page < row_count do
fetch_and_persist(
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
)
{:error, cause}
else
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
)
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp prepare_request(report_request, date_range, property, access_token) do
%GA4.ReportRequest{
report_request
| date_range: date_range,
property: property,
access_token: access_token,
offset: 0,
limit: @per_page
}
end
end

@ -1,244 +0,0 @@
defmodule Plausible.Google.GA4.HTTP do
@moduledoc """
HTTP client implementation for Google Analytics 4 API.
"""
alias Plausible.HTTPClient
require Logger
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
params = %{
requests: [
%{
property: report_request.property,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
metrics: Enum.map(report_request.metrics, &build_metric/1),
orderBys: [
%{
dimension: %{
dimensionName: "date",
orderType: "ALPHANUMERIC"
},
desc: true
}
],
limit: report_request.limit,
offset: report_request.offset
}
]
}
url =
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
response =
HTTPClient.impl().post(
url,
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
row_count <- Map.fetch!(report, "rowCount"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, row_count}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
)
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, reason} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
)
{:error, :request_failed}
end
end
defp build_metric(expression) do
case String.split(expression, " = ") do
[name, expression] ->
%{
name: name,
expression: expression
}
[name] ->
%{name: name}
end
end
defp parse_report_from_response(%{"reports" => [report | _]}) do
{:ok, report}
end
defp parse_report_from_response(body) do
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
Logger.error(
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
defp convert_to_maps(%{
"rows" => rows,
"dimensionHeaders" => dimension_headers,
"metricHeaders" => metric_headers
})
when is_list(rows) do
dimension_headers = Enum.map(dimension_headers, & &1["name"])
metric_headers = Enum.map(metric_headers, & &1["name"])
report =
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
dimension_values = Enum.map(dimensions, & &1["value"])
metric_values = Enum.map(metrics, & &1["value"])
metrics = Enum.zip(metric_headers, metric_values)
dimensions = Enum.zip(dimension_headers, dimension_values)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
{:error, {:invalid_response, response}}
end
def list_accounts_for_user(access_token) do
url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
{:error, :unknown}
end
end
def get_property(access_token, property) do
url = "#{admin_api_url()}/v1beta/#{property}"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
{:error, :not_found}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 property #{property}",
extra: %{error: error}
)
{:error, :unknown}
end
end
@earliest_valid_date "2015-08-14"
def get_analytics_start_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :start)
end
def get_analytics_end_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :end)
end
defp get_analytics_boundary_date(access_token, property, edge) do
descending? = edge == :end
params = %{
requests: [
%{
property: "#{property}",
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "date"}],
metrics: [%{name: "screenPageViews"}],
orderBys: [
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
],
limit: 1
}
]
}
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["rows"] do
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 #{edge} date",
extra: %{error: error}
)
{:error, :unknown}
end
end
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
end

@ -1,135 +0,0 @@
defmodule Plausible.Google.GA4.ReportRequest do
@moduledoc """
Report request struct for Google Analytics 4 API
"""
defstruct [
:dataset,
:dimensions,
:metrics,
:date_range,
:property,
:access_token,
:offset,
:limit
]
@type t() :: %__MODULE__{
dataset: String.t(),
dimensions: [String.t()],
metrics: [String.t()],
date_range: Date.Range.t(),
property: term(),
access_token: String.t(),
offset: non_neg_integer(),
limit: non_neg_integer()
}
def full_report do
[
%__MODULE__{
dataset: "imported_visitors",
dimensions: ["date"],
metrics: [
"totalUsers",
"screenPageViews",
"bounces = sessions - engagedSessions",
"sessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_sources",
dimensions: [
"date",
"sessionSource",
"sessionMedium",
"sessionCampaignName",
"sessionManualAdContent",
"sessionGoogleAdsKeyword",
"pageReferrer"
],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
},
%__MODULE__{
dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"userEngagementDuration",
"bounces = sessions - engagedSessions"
]
},
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
# %__MODULE__{
# dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"],
# metrics: [
# "totalUsers",
# "sessions",
# "screenPageViews",
# "userEngagementDuration",
# "bounces = sessions - engagedSessions"
# ]
# },
%__MODULE__{
dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_devices",
dimensions: ["date", "deviceCategory"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_browsers",
dimensions: ["date", "browser"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
}
]
end
end

@ -2,6 +2,97 @@ defmodule Plausible.Google.HTTP do
require Logger require Logger
alias Plausible.HTTPClient alias Plausible.HTTPClient
@spec get_report(Plausible.Google.ReportRequest.t()) ::
{:ok, {[map()], String.t() | nil}} | {:error, any()}
def get_report(%Plausible.Google.ReportRequest{} = report_request) do
params = %{
reportRequests: [
%{
viewId: report_request.view_id,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: report_request.page_size,
pageToken: report_request.page_token
}
]
}
response =
HTTPClient.impl().post(
"#{reporting_api_url()}/v4/reports:batchGet",
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
token <- Map.get(report, "nextPageToken"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, token}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, _reason} ->
{:error, :request_failed}
end
end
defp parse_report_from_response(body) do
with %{"reports" => [report | _]} <- body do
{:ok, report}
else
_ ->
Sentry.Context.set_extra_context(%{google_analytics_response: body})
Logger.error(
"Google Analytics: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
end
defp convert_to_maps(%{
"data" => %{} = data,
"columnHeader" => %{
"dimensions" => dimension_headers,
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
}
}) do
metric_headers = Enum.map(metric_headers, & &1["name"])
rows = Map.get(data, "rows", [])
report =
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
metrics = Enum.zip(metric_headers, metrics)
dimensions = Enum.zip(dimension_headers, dimensions)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics_response: response})
{:error, {:invalid_response, response}}
end
def list_sites(access_token) do def list_sites(access_token) do
url = "#{api_url()}/webmasters/v3/sites" url = "#{api_url()}/webmasters/v3/sites"
headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}] headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]
@ -22,7 +113,7 @@ defmodule Plausible.Google.HTTP do
end end
end end
def fetch_access_token!(code) do def fetch_access_token(code) do
url = "#{api_url()}/oauth2/v4/token" url = "#{api_url()}/oauth2/v4/token"
headers = [{"Content-Type", "application/x-www-form-urlencoded"}] headers = [{"Content-Type", "application/x-www-form-urlencoded"}]
@ -39,6 +130,24 @@ defmodule Plausible.Google.HTTP do
response.body response.body
end end
def list_views_for_user(access_token) do
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
{:error, :unknown}
end
end
def list_stats(access_token, property, date_range, limit, page \\ nil) do def list_stats(access_token, property, date_range, limit, page \\ nil) do
property = URI.encode_www_form(property) property = URI.encode_www_form(property)
@ -106,9 +215,57 @@ defmodule Plausible.Google.HTTP do
end end
end end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(view_id, access_token) do
params = %{
reportRequests: [
%{
viewId: view_id,
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "ga:date", histogramBuckets: []}],
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
pageSize: 1
}
]
}
url = "#{reporting_api_url()}/v4/reports:batchGet"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["data"]["rows"] do
[%{"dimensions" => [date_str]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)})
{:error, reason}
end
end
defp config, do: Application.get_env(:plausible, :google) defp config, do: Application.get_env(:plausible, :google)
defp client_id, do: Keyword.fetch!(config(), :client_id) defp client_id, do: Keyword.fetch!(config(), :client_id)
defp client_secret, do: Keyword.fetch!(config(), :client_secret) defp client_secret, do: Keyword.fetch!(config(), :client_secret)
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url) defp api_url, do: Keyword.fetch!(config(), :api_url)
defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback" defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
end end

@ -1,8 +1,4 @@
defmodule Plausible.Google.UA.ReportRequest do defmodule Plausible.Google.ReportRequest do
@moduledoc """
Report request struct for Universal Analytics API
"""
defstruct [ defstruct [
:dataset, :dataset,
:dimensions, :dimensions,

@ -1,155 +0,0 @@
defmodule Plausible.Google.UA.API do
@moduledoc """
API for Universal Analytics
"""
alias Plausible.Google
alias Plausible.Google.UA
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 7_500
@backoff_factor :timer.seconds(10)
@max_attempts 5
@spec list_views(access_token :: String.t()) ::
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
@doc """
Lists Google Analytics views grouped by hostname.
"""
def list_views(access_token) do
case UA.HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
views =
views
|> Enum.group_by(&view_hostname/1, &view_names/1)
|> Enum.sort_by(fn {key, _} -> key end)
{:ok, views}
error ->
error
end
end
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
@doc """
Returns a single Google Analytics view if the user has access to it.
"""
def get_view(access_token, lookup_id) do
with {:ok, views} <- list_views(access_token) do
views =
views
|> Enum.map(&elem(&1, 1))
|> List.flatten()
case Enum.find(views, fn {_name, id} -> id == lookup_id end) do
{view_name, view_id} ->
{:ok, %{id: view_id, name: "#{view_name}"}}
nil ->
{:error, :not_found}
end
end
end
def get_analytics_start_date(access_token, view_id) do
UA.HTTP.get_analytics_start_date(access_token, view_id)
end
def get_analytics_end_date(access_token, view_id) do
UA.HTTP.get_analytics_end_date(access_token, view_id)
end
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
This function fetches Google Analytics reports in batches of #{@per_page} per
request. The batches are then passed to persist callback.
Requests to Google Analytics can fail, and are retried at most
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
importing has finished or `{:error, term()}` when a request to GA failed too
many times.
Useful links:
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(date_range, view_id, auth, persist_fn) do
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
do_import_analytics(date_range, view_id, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
report_request = %UA.ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case UA.HTTP.get_report(report_request) do
{:ok, {rows, next_page_token}} ->
:ok = persist_fn.(report_request.dataset, rows)
if next_page_token do
fetch_and_persist(
%UA.ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
{:error, cause}
else
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp view_hostname(view) do
case view do
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
_any -> "Others"
end
end
defp view_names(%{"name" => name, "id" => id}) do
{"#{id} - #{name}", id}
end
end

@ -1,191 +0,0 @@
defmodule Plausible.Google.UA.HTTP do
@moduledoc """
HTTP client implementation for Universal Analytics API.
"""
require Logger
alias Plausible.HTTPClient
@spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
{:ok, {[map()], String.t() | nil}} | {:error, any()}
def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
params = %{
reportRequests: [
%{
viewId: report_request.view_id,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: report_request.page_size,
pageToken: report_request.page_token
}
]
}
response =
HTTPClient.impl().post(
"#{reporting_api_url()}/v4/reports:batchGet",
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
token <- Map.get(report, "nextPageToken"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, token}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, _reason} ->
{:error, :request_failed}
end
end
defp parse_report_from_response(%{"reports" => [report | _]}) do
{:ok, report}
end
defp parse_report_from_response(body) do
Sentry.Context.set_extra_context(%{universal_analytics_response: body})
Logger.error(
"Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
defp convert_to_maps(%{
"data" => %{} = data,
"columnHeader" => %{
"dimensions" => dimension_headers,
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
}
}) do
metric_headers = Enum.map(metric_headers, & &1["name"])
rows = Map.get(data, "rows", [])
report =
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
metrics = Enum.zip(metric_headers, metrics)
dimensions = Enum.zip(dimension_headers, dimensions)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{universal_analytics_response: response})
{:error, {:invalid_response, response}}
end
def list_views_for_user(access_token) do
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error listing UA views for user", extra: %{error: error})
{:error, :unknown}
end
end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :start)
end
def get_analytics_end_date(access_token, view_id) do
get_analytics_boundary_date(access_token, view_id, :end)
end
defp get_analytics_boundary_date(access_token, view_id, edge) do
sort_order =
if edge == :start do
"ASCENDING"
else
"DESCENDING"
end
params = %{
reportRequests: [
%{
viewId: view_id,
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "ga:date", histogramBuckets: []}],
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}],
pageSize: 1
}
]
}
url = "#{reporting_api_url()}/v4/reports:batchGet"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["data"]["rows"] do
[%{"dimensions" => [date_str]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving UA #{edge} date",
extra: %{error: error}
)
{:error, :unknown}
end
end
defp config, do: Application.get_env(:plausible, :google)
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url)
end

@ -37,9 +37,6 @@ defmodule Plausible.Imported do
# Maximum number of complete imports to account for when querying stats # Maximum number of complete imports to account for when querying stats
@max_complete_imports 5 @max_complete_imports 5
@spec schemas() :: [module()]
def schemas, do: @tables
@spec tables() :: [String.t()] @spec tables() :: [String.t()]
def tables, do: @table_names def tables, do: @table_names
@ -51,29 +48,28 @@ defmodule Plausible.Imported do
def load_import_data(site) do def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site) complete_import_ids = list_complete_import_ids(site)
dates = get_imports_date_range(site) earliest_import = get_earliest_import(site) || %{}
%{ %{
site site
| import_data_loaded: true, | import_data_loaded: true,
earliest_import_start_date: dates.start_date, earliest_import_start_date: Map.get(earliest_import, :start_date),
latest_import_end_date: dates.end_date, earliest_import_end_date: Map.get(earliest_import, :end_date),
complete_import_ids: complete_import_ids complete_import_ids: complete_import_ids
} }
end end
@spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil @spec get_import(non_neg_integer()) :: SiteImport.t() | nil
def get_import(site, import_id) do def get_import(import_id) do
Repo.get_by(SiteImport, id: import_id, site_id: site.id) Repo.get(SiteImport, import_id)
end end
defdelegate listen(), to: Imported.Importer defdelegate listen(), to: Imported.Importer
@spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()] @spec list_all_imports(Site.t()) :: [SiteImport.t()]
def list_all_imports(site, status \\ nil) do def list_all_imports(site) do
imports = imports =
from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at]) from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
|> maybe_filter_by_status(status)
|> Repo.all() |> Repo.all()
if site.imported_data && not Enum.any?(imports, & &1.legacy) do if site.imported_data && not Enum.any?(imports, & &1.legacy) do
@ -83,72 +79,42 @@ defmodule Plausible.Imported do
end end
end end
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
def other_imports_in_progress?(site_import) do
Repo.exists?(
from(i in SiteImport,
where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
)
)
end
defp maybe_filter_by_status(query, nil), do: query
defp maybe_filter_by_status(query, status) do
where(query, [i], i.status == ^status)
end
@spec list_complete_import_ids(Site.t()) :: [non_neg_integer()] @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()]
def list_complete_import_ids(site) do def list_complete_import_ids(site) do
ids = ids =
from(i in SiteImport, from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(), where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: {i.legacy, i.id}, select: i.id,
limit: @max_complete_imports limit: @max_complete_imports
) )
|> Repo.all() |> Repo.all()
has_legacy? = Enum.any?(ids, fn {legacy?, _} -> legacy? end)
ids = Enum.map(ids, &elem(&1, 1))
# account for legacy imports as well # account for legacy imports as well
if has_legacy? || (site.imported_data && site.imported_data.status == "ok") do if site.imported_data && site.imported_data.status == "ok" do
[0 | ids] [0 | ids]
else else
ids ids
end end
end end
@spec get_imports_date_range(Site.t()) :: %{ @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
start_date: Date.t() | nil, def get_earliest_import(site) do
end_date: Date.t() | nil first_import =
}
def get_imports_date_range(site) do
dates =
from(i in SiteImport, from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(), where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
select: %{start_date: min(i.start_date), end_date: max(i.end_date)} order_by: i.start_date,
limit: 1
) )
|> Repo.one() |> Repo.one()
dates = dates || %{start_date: nil, end_date: nil} legacy_import =
if site.imported_data && site.imported_data.status == "ok" do
if site.imported_data && site.imported_data.status == "ok" do site.imported_data
start_date = end
[dates.start_date, site.imported_data.start_date]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
end_date =
[dates.end_date, site.imported_data.end_date]
|> Enum.reject(&is_nil/1)
|> Enum.max(Date, fn -> nil end)
%{start_date: start_date, end_date: end_date} [legacy_import, first_import]
else |> Enum.reject(&is_nil/1)
dates |> Enum.min_by(& &1.start_date, Date, fn -> nil end)
end
end end
@spec delete_imports_for_site(Site.t()) :: :ok @spec delete_imports_for_site(Site.t()) :: :ok
@ -157,66 +123,4 @@ defmodule Plausible.Imported do
:ok :ok
end end
@spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) ::
{:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window}
def check_dates(_site, nil, _end_date), do: {:error, :no_data}
def check_dates(site, start_date, end_date) do
cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
end_date = Enum.min([end_date, cutoff_date], Date)
with true <- Date.diff(end_date, start_date) >= 2,
[_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do
longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first))
{:ok, longest.first, longest.last}
else
_ -> {:error, :no_time_window}
end
end
defp find_free_ranges(start_date, end_date, site) do
occupied_ranges =
site
|> Imported.list_all_imports(Imported.SiteImport.completed())
|> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2))
|> Enum.map(&Date.range(&1.start_date, &1.end_date))
Date.range(start_date, end_date)
|> free_ranges(start_date, occupied_ranges, [])
end
# This function recursively finds open ranges that are not yet occupied
# by existing imported data. The idea is that we keep moving a dynamic
# date index `d` from start until the end of `imported_range`, hopping
# over each occupied range, and capturing the open ranges step-by-step
# in the `result` array.
defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do
cond do
Date.diff(occupied_range.last, d) <= 0 ->
free_ranges(import_range, d, rest_of_occupied_ranges, result)
in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 ->
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
true ->
free_range = Date.range(d, occupied_range.first)
result = result ++ [free_range]
d = occupied_range.last
free_ranges(import_range, d, rest_of_occupied_ranges, result)
end
end
defp free_ranges(import_range, d, [], result) do
if Date.diff(import_range.last, d) < 2 do
result
else
result ++ [Date.range(d, import_range.last)]
end
end
defp in_range?(date, range) do
Date.before?(range.first, date) && Date.after?(range.last, date)
end
end end

@ -8,11 +8,9 @@ defmodule Plausible.Imported.Browser do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :browser, :string field :browser, :string
field :browser_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -8,13 +8,12 @@ defmodule Plausible.Imported.Buffer do
use GenServer use GenServer
require Logger require Logger
def start_link(opts \\ []) do def start_link do
GenServer.start_link(__MODULE__, opts) GenServer.start_link(__MODULE__, nil)
end end
def init(opts) do def init(_opts) do
flush_interval = Keyword.get(opts, :flush_interval_ms, 1000) {:ok, %{buffers: %{}}}
{:ok, %{flush_interval: flush_interval, buffers: %{}}}
end end
@spec insert_many(pid(), term(), [map()]) :: :ok @spec insert_many(pid(), term(), [map()]) :: :ok
@ -69,14 +68,14 @@ defmodule Plausible.Imported.Buffer do
def handle_call(:flush_all_buffers, _from, state) do def handle_call(:flush_all_buffers, _from, state) do
Enum.each(state.buffers, fn {table_name, records} -> Enum.each(state.buffers, fn {table_name, records} ->
flush_buffer(records, table_name, state.flush_interval) flush_buffer(records, table_name)
end) end)
{:reply, :ok, put_in(state.buffers, %{})} {:reply, :ok, put_in(state.buffers, %{})}
end end
def handle_continue({:flush, table_name}, state) do def handle_continue({:flush, table_name}, state) do
flush_buffer(state.buffers[table_name], table_name, state.flush_interval) flush_buffer(state.buffers[table_name], table_name)
{:noreply, put_in(state.buffers[table_name], [])} {:noreply, put_in(state.buffers[table_name], [])}
end end
@ -86,10 +85,10 @@ defmodule Plausible.Imported.Buffer do
|> Keyword.fetch!(:max_buffer_size) |> Keyword.fetch!(:max_buffer_size)
end end
defp flush_buffer(records, table_name, flush_interval) do defp flush_buffer(records, table_name) do
# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
# sleep call slows down the flushing # sleep call slows down the flushing
Process.sleep(flush_interval) Process.sleep(1000)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer") Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
insert_all(table_name, records) insert_all(table_name, records)

@ -20,13 +20,7 @@ defmodule Plausible.Imported.CSVImporter do
@impl true @impl true
def import_data(site_import, opts) do def import_data(site_import, opts) do
%{ %{id: import_id, site_id: site_id} = site_import
id: import_id,
site_id: site_id,
start_date: start_date,
end_date: end_date
} = site_import
uploads = Keyword.fetch!(opts, :uploads) uploads = Keyword.fetch!(opts, :uploads)
%{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} = %{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} =
@ -37,44 +31,52 @@ defmodule Plausible.Imported.CSVImporter do
|> Keyword.replace!(:pool_size, 1) |> Keyword.replace!(:pool_size, 1)
|> Ch.start_link() |> Ch.start_link()
Enum.each(uploads, fn upload -> ranges =
%{"filename" => filename, "s3_url" => s3_url} = upload Enum.map(uploads, fn upload ->
%{"filename" => filename, "s3_url" => s3_url} = upload
{table, _, _} = parse_filename!(filename)
s3_structure = input_structure!(table) ".csv" = Path.extname(filename)
table = Path.rootname(filename)
s3_structure_cols_expr = ensure_importable_table!(table)
s3_structure
|> String.split(",", trim: true) s3_structure = input_structure!(table)
|> Enum.map_join(", ", fn kv ->
[col, _type] = String.split(kv) statement =
col """
end) INSERT INTO {table:Identifier} \
SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \
statement = FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\
""" """
INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \
SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ params =
FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \ %{
WHERE date >= {start_date:Date} AND date <= {end_date:Date}\ "table" => table,
""" "site_id" => site_id,
"import_id" => import_id,
params = "s3_url" => s3_url,
%{ "s3_access_key_id" => s3_access_key_id,
"table" => table, "s3_secret_access_key" => s3_secret_access_key,
"site_id" => site_id, "s3_format" => "CSVWithNames",
"import_id" => import_id, "s3_structure" => s3_structure
"s3_url" => s3_url, }
"s3_access_key_id" => s3_access_key_id,
"s3_secret_access_key" => s3_secret_access_key, Ch.query!(ch, statement, params, timeout: :infinity)
"s3_format" => "CSVWithNames",
"s3_structure" => s3_structure, %Ch.Result{rows: [[min_date, max_date]]} =
"start_date" => start_date, Ch.query!(
"end_date" => end_date ch,
} "SELECT min(date), max(date) FROM {table:Identifier} WHERE site_id = {site_id:UInt64} AND import_id = {import_id:UInt64}",
%{"table" => table, "site_id" => site_id, "import_id" => import_id}
Ch.query!(ch, statement, params, timeout: :infinity) )
end)
Date.range(min_date, max_date)
end)
{:ok,
%{
start_date: Enum.min_by(ranges, & &1.first, Date).first,
end_date: Enum.max_by(ranges, & &1.last, Date).last
}}
rescue rescue
# we are cancelling on any argument or ClickHouse errors # we are cancelling on any argument or ClickHouse errors
e in [ArgumentError, Ch.Error] -> e in [ArgumentError, Ch.Error] ->
@ -83,147 +85,30 @@ defmodule Plausible.Imported.CSVImporter do
input_structures = %{ input_structures = %{
"imported_browsers" => "imported_browsers" =>
"date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, browser String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_devices" => "imported_devices" =>
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_entry_pages" => "imported_entry_pages" =>
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32",
"imported_exit_pages" => "imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64",
"date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
"imported_locations" => "imported_locations" =>
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_operating_systems" => "imported_operating_systems" =>
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", "date Date, operating_system String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_pages" => "imported_pages" =>
"date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64", "date Date, hostname String, page String, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
"imported_sources" => "imported_sources" =>
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" => "imported_visitors" =>
"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64" "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
} }
@doc """
Extracts min/max date range from a list of uploads.
Examples:
iex> date_range([
...> %{"filename" => "imported_devices_20190101_20210101.csv"},
...> "imported_pages_20200101_20220101.csv"
...> ])
Date.range(~D[2019-01-01], ~D[2022-01-01])
iex> date_range([])
nil
"""
@spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
def date_range([]), do: nil
defp date_range([upload | uploads], prev_start_date, prev_end_date) do
filename =
case upload do
%{"filename" => filename} -> filename
filename when is_binary(filename) -> filename
end
{_table, start_date, end_date} = parse_filename!(filename)
start_date =
if prev_start_date do
Enum.min([start_date, prev_start_date], Date)
else
start_date
end
end_date =
if prev_end_date do
Enum.max([end_date, prev_end_date], Date)
else
end_date
end
date_range(uploads, start_date, end_date)
end
defp date_range([], first, last), do: Date.range(first, last)
@spec parse_date!(String.t()) :: Date.t()
defp parse_date!(date) do
date |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date()
end
@doc """
Extracts table name and min/max dates from the filename.
Examples:
iex> parse_filename!("my_data.csv")
** (ArgumentError) invalid filename
iex> parse_filename!("imported_devices_00010101_20250101.csv")
{"imported_devices", ~D[0001-01-01], ~D[2025-01-01]}
"""
@spec parse_filename!(String.t()) ::
{table :: String.t(), start_date :: Date.t(), end_date :: Date.t()}
def parse_filename!(filename)
for {table, input_structure} <- input_structures do for {table, input_structure} <- input_structures do
defp input_structure!(unquote(table)), do: unquote(input_structure) defp input_structure!(unquote(table)), do: unquote(input_structure)
defp ensure_importable_table!(unquote(table)), do: :ok
def parse_filename!(
<<unquote(table)::bytes, ?_, start_date::8-bytes, ?_, end_date::8-bytes, ".csv">>
) do
{unquote(table), parse_date!(start_date), parse_date!(end_date)}
end
end
def parse_filename!(_filename) do
raise ArgumentError, "invalid filename"
end
@doc """
Checks if the provided filename conforms to the expected format.
Examples:
iex> valid_filename?("my_data.csv")
false
iex> valid_filename?("imported_devices_00010101_20250101.csv")
true
"""
@spec valid_filename?(String.t()) :: boolean
def valid_filename?(filename) do
try do
parse_filename!(filename)
else
_ -> true
rescue
_ -> false
end
end end
@doc """ defp ensure_importable_table!(table) do
Extracts the table name from the provided filename. raise ArgumentError, "table #{table} is not supported for data import"
Raises if the filename doesn't conform to the expected format.
Examples:
iex> extract_table("my_data.csv")
** (ArgumentError) invalid filename
iex> extract_table("imported_devices_00010101_20250101.csv")
"imported_devices"
"""
@spec extract_table(String.t()) :: String.t()
def extract_table(filename) do
{table, _start_date, _end_date} = parse_filename!(filename)
table
end end
end end

@ -11,7 +11,6 @@ defmodule Plausible.Imported.Device do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -11,7 +11,6 @@ defmodule Plausible.Imported.EntryPage do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :entrances, Ch, type: "UInt64" field :entrances, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -8,10 +8,7 @@ defmodule Plausible.Imported.ExitPage do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :exit_page, :string field :exit_page, :string
field :exits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end end
end end

@ -1,276 +0,0 @@
defmodule Plausible.Imported.GoogleAnalytics4 do
@moduledoc """
Import implementation for Google Analytics 4.
"""
use Plausible.Imported.Importer
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@impl true
def name(), do: :google_analytics_4
@impl true
def label(), do: "Google Analytics 4"
@impl true
def email_template(), do: "google_analytics_import.html"
@impl true
def parse_args(
%{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
) do
start_date = Date.from_iso8601!(start_date)
end_date = Date.from_iso8601!(end_date)
date_range = Date.range(start_date, end_date)
auth = {
Map.fetch!(args, "access_token"),
Map.fetch!(args, "refresh_token"),
Map.fetch!(args, "token_expires_at")
}
[
property: property,
date_range: date_range,
auth: auth
]
end
@doc """
Imports stats from a Google Analytics 4 property to a Plausible site.
This function fetches Google Analytics 4 reports which are then passed in batches
to Clickhouse by the `Plausible.Imported.Buffer` process.
"""
@impl true
def import_data(site_import, opts) do
date_range = Keyword.fetch!(opts, :date_range)
property = Keyword.fetch!(opts, :property)
auth = Keyword.fetch!(opts, :auth)
flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
{:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
persist_fn = fn table, rows ->
records = from_report(rows, site_import.site_id, site_import.id, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
try do
Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
after
Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer)
end
end
def from_report(nil, _site_id, _import_id, _metric), do: nil
def from_report(data, site_id, import_id, table) do
Enum.reduce(data, [], fn row, acc ->
if Map.get(row.dimensions, "date") in @missing_values do
acc
else
[new_from_report(site_id, import_id, table, row) | acc]
end
end)
end
defp parse_number(nr) do
{float, ""} = Float.parse(nr)
round(float)
end
defp new_from_report(site_id, import_id, "imported_visitors", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_sources", row) do
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
referrer =
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
end
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
referrer: referrer,
# Only `source` exists in GA4 API
utm_source: nil,
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently
exits: 0,
time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
entry_page: row.dimensions |> Map.fetch!("landingPage"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
}
end
# NOTE: no exit pages metrics in GA4 API available for now
# defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
# %{
# site_id: site_id,
# import_id: import_id,
# date: get_date(row),
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
# }
# end
defp new_from_report(site_id, import_id, "imported_locations", row) do
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
city_data = Location.get_city(city_name, country_code)
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
country: country_code,
region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
city: city_data && city_data.id,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_devices", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
@browser_google_to_plausible %{
"User-Agent:Opera" => "Opera",
"Mozilla Compatible Agent" => "Mobile App",
"Android Webview" => "Mobile App",
"Android Browser" => "Mobile App",
"Safari (in-app)" => "Mobile App",
"User-Agent: Mozilla" => "Firefox",
"(not set)" => ""
}
defp new_from_report(site_id, import_id, "imported_browsers", row) do
browser = Map.fetch!(row.dimensions, "browser")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser),
# Does not exist in GA4 API
browser_version: nil,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
@os_google_to_plausible %{
"Macintosh" => "Mac",
"Linux" => "GNU/Linux",
"(not set)" => ""
}
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
os = Map.fetch!(row.dimensions, "operatingSystem")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os),
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp get_date(%{dimensions: %{"date" => date}}) do
date
|> Timex.parse!("%Y%m%d", :strftime)
|> NaiveDateTime.to_date()
end
defp default_if_missing(value, default \\ nil)
defp default_if_missing(value, default) when value in @missing_values, do: default
defp default_if_missing(value, _default), do: value
defp parse_referrer(nil), do: nil
defp parse_referrer("(direct)"), do: nil
defp parse_referrer("google"), do: "Google"
defp parse_referrer("bing"), do: "Bing"
defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
defp parse_referrer(ref) do
RefInspector.parse("https://" <> ref)
|> PlausibleWeb.RefInspector.parse()
end
end

@ -4,7 +4,6 @@ defmodule Plausible.Imported.ImportSources do
""" """
@sources [ @sources [
Plausible.Imported.GoogleAnalytics4,
Plausible.Imported.UniversalAnalytics, Plausible.Imported.UniversalAnalytics,
Plausible.Imported.NoopImporter, Plausible.Imported.NoopImporter,
Plausible.Imported.CSVImporter Plausible.Imported.CSVImporter

@ -91,7 +91,6 @@ defmodule Plausible.Imported.Importer do
like LiveView, where notifications can be listened for via `handle_info/2`. like LiveView, where notifications can be listened for via `handle_info/2`.
""" """
alias Plausible.Imported
alias Plausible.Imported.SiteImport alias Plausible.Imported.SiteImport
alias Plausible.Repo alias Plausible.Repo
@ -99,7 +98,7 @@ defmodule Plausible.Imported.Importer do
@callback label() :: String.t() @callback label() :: String.t()
@callback email_template() :: String.t() @callback email_template() :: String.t()
@callback parse_args(map()) :: Keyword.t() @callback parse_args(map()) :: Keyword.t()
@callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()} @callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()} @callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
@callback on_success(SiteImport.t(), map()) :: :ok @callback on_success(SiteImport.t(), map()) :: :ok
@callback on_failure(SiteImport.t()) :: :ok @callback on_failure(SiteImport.t()) :: :ok
@ -109,7 +108,7 @@ defmodule Plausible.Imported.Importer do
@behaviour Plausible.Imported.Importer @behaviour Plausible.Imported.Importer
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) :: @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress} {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
def new_import(site, user, opts) do def new_import(site, user, opts) do
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1) Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
end end
@ -156,7 +155,7 @@ defmodule Plausible.Imported.Importer do
def new_import(source, site, user, opts, before_start_fun) do def new_import(source, site, user, opts, before_start_fun) do
import_params = import_params =
opts opts
|> Keyword.take([:label, :start_date, :end_date, :legacy]) |> Keyword.take([:start_date, :end_date, :legacy])
|> Keyword.put(:source, source) |> Keyword.put(:source, source)
|> Map.new() |> Map.new()
@ -223,15 +222,11 @@ defmodule Plausible.Imported.Importer do
:ok = listen() :ok = listen()
end end
if not Imported.other_imports_in_progress?(site_import) do opts
opts |> Keyword.put(:import_id, site_import.id)
|> Keyword.put(:import_id, site_import.id) |> Map.new()
|> Map.new() |> Plausible.Workers.ImportAnalytics.new()
|> Plausible.Workers.ImportAnalytics.new() |> Oban.insert()
|> Oban.insert()
else
{:error, :import_in_progress}
end
end end
defp mark_complete(site_import, extra_data, on_success_fun) do defp mark_complete(site_import, extra_data, on_success_fun) do

@ -13,7 +13,6 @@ defmodule Plausible.Imported.Location do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -8,11 +8,9 @@ defmodule Plausible.Imported.OperatingSystem do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :operating_system, :string field :operating_system, :string
field :operating_system_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -9,7 +9,6 @@ defmodule Plausible.Imported.Page do
field :date, :date field :date, :date
field :hostname, :string field :hostname, :string
field :page, :string field :page, :string
field :visits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64" field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"

@ -18,7 +18,6 @@ defmodule Plausible.Imported.SiteImport do
schema "site_imports" do schema "site_imports" do
field :start_date, :date field :start_date, :date
field :end_date, :date field :end_date, :date
field :label, :string
field :source, Ecto.Enum, values: ImportSources.names() field :source, Ecto.Enum, values: ImportSources.names()
field :status, Ecto.Enum, values: @statuses field :status, Ecto.Enum, values: @statuses
field :legacy, :boolean, default: false field :legacy, :boolean, default: false
@ -33,14 +32,11 @@ defmodule Plausible.Imported.SiteImport do
defmacro unquote(status)(), do: unquote(status) defmacro unquote(status)(), do: unquote(status)
end end
@spec label(t() | Site.ImportedData.t()) :: String.t()
def label(%__MODULE__{source: source, label: label}) do
build_label(ImportSources.by_name(source).label(), label)
end
# NOTE: this is necessary for backwards compatibility # NOTE: this is necessary for backwards compatibility
# with legacy imports # with legacy imports
def label(%Site.ImportedData{source: source}), do: build_label(source, nil) @spec label(t() | Site.ImportedData.t()) :: String.t()
def label(%__MODULE__{source: source}), do: ImportSources.by_name(source).label()
def label(%Site.ImportedData{source: source}), do: source
@spec from_legacy(Site.ImportedData.t()) :: t() @spec from_legacy(Site.ImportedData.t()) :: t()
def from_legacy(%Site.ImportedData{} = data) do def from_legacy(%Site.ImportedData{} = data) do
@ -64,7 +60,7 @@ defmodule Plausible.Imported.SiteImport do
@spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t() @spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
def create_changeset(site, user, params) do def create_changeset(site, user, params) do
%__MODULE__{} %__MODULE__{}
|> cast(params, [:label, :source, :start_date, :end_date, :legacy]) |> cast(params, [:source, :start_date, :end_date, :legacy])
|> validate_required([:source]) |> validate_required([:source])
|> put_assoc(:site, site) |> put_assoc(:site, site)
|> put_assoc(:imported_by, user) |> put_assoc(:imported_by, user)
@ -89,7 +85,4 @@ defmodule Plausible.Imported.SiteImport do
def fail_changeset(site_import) do def fail_changeset(site_import) do
change(site_import, status: failed()) change(site_import, status: failed())
end end
defp build_label(source, nil), do: source
defp build_label(source, label), do: "#{source} (#{label})"
end end

@ -8,8 +8,6 @@ defmodule Plausible.Imported.Source do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :source, :string field :source, :string
field :referrer, :string
field :utm_source, :string
field :utm_medium, :string field :utm_medium, :string
field :utm_campaign, :string field :utm_campaign, :string
field :utm_content, :string field :utm_content, :string
@ -17,7 +15,6 @@ defmodule Plausible.Imported.Source do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics do
end end
try do try do
Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn) Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn)
after after
Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer) Plausible.Imported.Buffer.stop(buffer)

@ -29,7 +29,6 @@ defmodule Plausible.Ingestion.Event do
| :dc_ip | :dc_ip
| :site_ip_blocklist | :site_ip_blocklist
| :site_country_blocklist | :site_country_blocklist
| :site_page_blocklist
@type t() :: %__MODULE__{ @type t() :: %__MODULE__{
domain: String.t() | nil, domain: String.t() | nil,
@ -44,6 +43,26 @@ defmodule Plausible.Ingestion.Event do
changeset: %Ecto.Changeset{} changeset: %Ecto.Changeset{}
} }
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
@spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} @spec build_and_buffer(Request.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}}
def build_and_buffer(%Request{domains: domains} = request) do def build_and_buffer(%Request{domains: domains} = request) do
processed_events = processed_events =
@ -80,10 +99,6 @@ defmodule Plausible.Ingestion.Event do
[:plausible, :ingest, :event, :dropped] [:plausible, :ingest, :event, :dropped]
end end
def telemetry_pipeline_step_duration() do
[:plausible, :ingest, :pipeline, :step]
end
@spec emit_telemetry_buffered(t()) :: :ok @spec emit_telemetry_buffered(t()) :: :ok
def emit_telemetry_buffered(event) do def emit_telemetry_buffered(event) do
:telemetry.execute(telemetry_event_buffered(), %{}, %{ :telemetry.execute(telemetry_event_buffered(), %{}, %{
@ -103,42 +118,33 @@ defmodule Plausible.Ingestion.Event do
defp pipeline() do defp pipeline() do
[ [
drop_datacenter_ip: &drop_datacenter_ip/1, &drop_datacenter_ip/1,
drop_shield_rule_page: &drop_shield_rule_page/1, &drop_shield_rule_ip/1,
drop_shield_rule_ip: &drop_shield_rule_ip/1, &put_geolocation/1,
put_geolocation: &put_geolocation/1, &drop_shield_rule_country/1,
drop_shield_rule_country: &drop_shield_rule_country/1, &put_user_agent/1,
put_user_agent: &put_user_agent/1, &put_basic_info/1,
put_basic_info: &put_basic_info/1, &put_referrer/1,
put_referrer: &put_referrer/1, &put_utm_tags/1,
put_utm_tags: &put_utm_tags/1, &put_props/1,
put_props: &put_props/1, &put_revenue/1,
put_revenue: &put_revenue/1, &put_salts/1,
put_salts: &put_salts/1, &put_user_id/1,
put_user_id: &put_user_id/1, &validate_clickhouse_event/1,
validate_clickhouse_event: &validate_clickhouse_event/1, &register_session/1,
register_session: &register_session/1, &write_to_buffer/1
write_to_buffer: &write_to_buffer/1
] ]
end end
defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do
Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event -> Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( case pipeline_step.(acc_event) do
telemetry_pipeline_step_duration(), %__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
fn -> execute_step(step_fn, acc_event) end, %__MODULE__{dropped?: false} = event -> {:cont, event}
%{step: "#{step_name}"} end
)
end) end)
end end
defp execute_step(step_fn, acc_event) do
case step_fn.(acc_event) do
%__MODULE__{dropped?: true} = dropped -> {:halt, dropped}
%__MODULE__{dropped?: false} = event -> {:cont, event}
end
end
defp new(domain, request) do defp new(domain, request) do
struct!(__MODULE__, domain: domain, request: request) struct!(__MODULE__, domain: domain, request: request)
end end
@ -154,6 +160,7 @@ defmodule Plausible.Ingestion.Event do
|> Keyword.put(:drop_reason, reason) |> Keyword.put(:drop_reason, reason)
emit_telemetry_dropped(event, reason) emit_telemetry_dropped(event, reason)
IO.inspect(reason, label: :dropped)
struct!(event, fields) struct!(event, fields)
end end
@ -176,18 +183,15 @@ defmodule Plausible.Ingestion.Event do
end end
defp drop_shield_rule_ip(%__MODULE__{} = event) do defp drop_shield_rule_ip(%__MODULE__{} = event) do
if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do domain = event.domain
drop(event, :site_ip_blocklist) address = event.request.remote_ip
else
event
end
end
defp drop_shield_rule_page(%__MODULE__{} = event) do case Plausible.Shield.IPRuleCache.get({domain, address}) do
if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do %Plausible.Shield.IPRule{action: :deny} ->
drop(event, :site_page_blocklist) drop(event, :site_ip_blocklist)
else
event _ ->
event
end end
end end
@ -260,10 +264,12 @@ defmodule Plausible.Ingestion.Event do
%__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event
) )
when is_binary(domain) and is_binary(cc) do when is_binary(domain) and is_binary(cc) do
if Plausible.Shields.country_blocked?(domain, cc) do case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do
drop(event, :site_country_blocklist) %Plausible.Shield.CountryRule{action: :deny} ->
else drop(event, :site_country_blocklist)
event
_ ->
event
end end
end end
@ -339,7 +345,8 @@ defmodule Plausible.Ingestion.Event do
%{ %{
event event
| clickhouse_event: ClickhouseEventV2.merge_session(event.clickhouse_event, session) | clickhouse_event:
Map.merge(event.clickhouse_event, Map.take(session, @session_properties))
} }
end end
@ -375,14 +382,18 @@ defmodule Plausible.Ingestion.Event do
uri = URI.parse(ref.referer) uri = URI.parse(ref.referer)
if PlausibleWeb.RefInspector.right_uri?(uri) do if PlausibleWeb.RefInspector.right_uri?(uri) do
PlausibleWeb.RefInspector.format_referrer(uri) host = String.replace_prefix(uri.host, "www.", "")
path = uri.path || ""
host <> String.trim_trailing(path, "/")
end end
end end
defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do
Plausible.Cache.Adapter.get(:user_agents, user_agent, fn -> case Cachex.fetch(:user_agents, user_agent, &UAInspector.parse/1) do
UAInspector.parse(user_agent) {:ok, user_agent} -> user_agent
end) {:commit, user_agent} -> user_agent
_ -> nil
end
end end
defp parse_user_agent(request), do: request defp parse_user_agent(request), do: request

@ -138,6 +138,7 @@ defmodule Plausible.Ingestion.Request do
if is_binary(referrer) do if is_binary(referrer) do
referrer = String.slice(referrer, 0..(@max_url_size - 1)) referrer = String.slice(referrer, 0..(@max_url_size - 1))
Changeset.put_change(changeset, :referrer, referrer) Changeset.put_change(changeset, :referrer, referrer)
else else
changeset changeset

@ -3,151 +3,8 @@ defmodule Plausible.S3 do
Helper functions for S3 exports/imports. Helper functions for S3 exports/imports.
""" """
@doc """
Returns the pre-configured S3 bucket for CSV exports.
config :plausible, Plausible.S3,
exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET")
Example:
iex> exports_bucket()
"test-exports"
"""
@spec exports_bucket :: String.t()
def exports_bucket, do: config(:exports_bucket)
@doc """
Returns the pre-configured S3 bucket for CSV imports.
config :plausible, Plausible.S3,
imports_bucket: System.fetch_env!("S3_IMPORTS_BUCKET")
Example:
iex> imports_bucket()
"test-imports"
"""
@spec imports_bucket :: String.t()
def imports_bucket, do: config(:imports_bucket)
defp config, do: Application.fetch_env!(:plausible, __MODULE__)
defp config(key), do: Keyword.fetch!(config(), key)
@doc """
Presigns an upload for an imported file.
In the current implementation the bucket always goes into the path component.
Example:
iex> %{
...> s3_url: "http://localhost:10000/test-imports/123/imported_browsers.csv",
...> presigned_url: "http://localhost:10000/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin" <> _
...> } = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")
"""
def import_presign_upload(site_id, filename) do
config = ExAws.Config.new(:s3)
s3_path = Path.join(to_string(site_id), filename)
bucket = imports_bucket()
{:ok, presigned_url} = ExAws.S3.presigned_url(config, :put, bucket, s3_path)
%{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url}
end
# to make ClickHouse see MinIO in dev and test envs we replace
# the host in the S3 URL with whatever's set in S3_CLICKHOUSE_HOST env var
if Mix.env() in [:dev, :test, :small_dev, :small_test] do
defp extract_s3_url(presigned_url) do
[s3_url, _] = String.split(presigned_url, "?")
if ch_host = System.get_env("S3_CLICKHOUSE_HOST") do
URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})
else
s3_url
end
end
else
defp extract_s3_url(presigned_url) do
[s3_url, _] = String.split(presigned_url, "?")
s3_url
end
end
@doc """
Chunks and uploads Zip archive to the provided S3 destination.
Returns a presigned URL to download the exported Zip archive from S3.
The URL expires in 24 hours.
In the current implementation the bucket always goes into the path component.
"""
@spec export_upload_multipart(Enumerable.t(), String.t(), Path.t(), String.t(), keyword) ::
:uri_string.uri_string()
def export_upload_multipart(stream, s3_bucket, s3_path, filename, config_overrides \\ []) do
config = ExAws.Config.new(:s3)
encoded_filename = URI.encode(filename)
disposition = ~s[attachment; filename="#{encoded_filename}"]
disposition =
if encoded_filename != filename do
disposition <> "; filename*=utf-8''#{encoded_filename}"
else
disposition
end
# 5 MiB is the smallest chunk size AWS S3 supports
chunk_into_parts(stream, 5 * 1024 * 1024)
|> ExAws.S3.upload(s3_bucket, s3_path,
content_disposition: disposition,
content_type: "application/zip"
)
|> ExAws.request!(config_overrides)
{:ok, download_url} =
ExAws.S3.presigned_url(config, :get, s3_bucket, s3_path, expires_in: _24hr = 86_400)
download_url
end
defp chunk_into_parts(stream, min_part_size) do
Stream.chunk_while(
stream,
_acc = %{buffer_size: 0, buffer: [], min_part_size: min_part_size},
_chunk_fun = &buffer_until_big_enough/2,
_after_fun = &flush_leftovers/1
)
end
defp buffer_until_big_enough(data, acc) do
%{buffer_size: prev_buffer_size, buffer: prev_buffer, min_part_size: min_part_size} = acc
new_buffer_size = prev_buffer_size + IO.iodata_length(data)
new_buffer = [prev_buffer | data]
if new_buffer_size > min_part_size do
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
{:cont, IO.iodata_to_binary(new_buffer), %{acc | buffer_size: 0, buffer: []}}
else
{:cont, %{acc | buffer_size: new_buffer_size, buffer: new_buffer}}
end
end
defp flush_leftovers(acc) do
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
{:cont, IO.iodata_to_binary(acc.buffer), %{acc | buffer_size: 0, buffer: []}}
end
@doc """ @doc """
Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3. Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3.
Example:
iex> import_clickhouse_credentials()
%{access_key_id: "minioadmin", secret_access_key: "minioadmin"}
""" """
@spec import_clickhouse_credentials :: @spec import_clickhouse_credentials ::
%{access_key_id: String.t(), secret_access_key: String.t()} %{access_key_id: String.t(), secret_access_key: String.t()}

@ -5,6 +5,17 @@ defmodule Plausible.Sentry.Client do
require Logger require Logger
@doc """
The Sentry.HTTPClient behaviour requires a child spec to be supplied.
In this case we don't want Sentry to manage our Finch instances, hence it's fed
with a dummy module for the sake of the contract.
XXX: Submit a Sentry PR making the child spec callback optional.
"""
def child_spec do
Task.child_spec(fn -> :noop end)
end
def post(url, headers, body) do def post(url, headers, body) do
req_opts = Application.get_env(:plausible, __MODULE__)[:finch_request_opts] || [] req_opts = Application.get_env(:plausible, __MODULE__)[:finch_request_opts] || []

@ -19,22 +19,27 @@ defmodule Plausible.Session.CacheStore do
defp find_session(_domain, nil), do: nil defp find_session(_domain, nil), do: nil
defp find_session(event, user_id) do defp find_session(event, user_id) do
from_cache = Plausible.Cache.Adapter.get(:sessions, {event.site_id, user_id}) from_cache = Cachex.get(:sessions, {event.site_id, user_id})
case from_cache do case from_cache do
nil -> {:ok, nil} ->
nil nil
session -> {:ok, session} ->
if Timex.diff(event.timestamp, session.timestamp, :minutes) <= 30 do if Timex.diff(event.timestamp, session.timestamp, :minutes) <= 30 do
session session
end end
{:error, e} ->
Sentry.capture_message("Cachex error", extra: %{error: e})
nil
end end
end end
defp persist_session(session) do defp persist_session(session) do
key = {session.site_id, session.user_id} key = {session.site_id, session.user_id}
Plausible.Cache.Adapter.put(:sessions, key, session) Cachex.put(:sessions, key, session, ttl: :timer.minutes(30))
session
end end
defp update_session(session, event) do defp update_session(session, event) do
@ -42,19 +47,7 @@ defmodule Plausible.Session.CacheStore do
session session
| user_id: event.user_id, | user_id: event.user_id,
timestamp: event.timestamp, timestamp: event.timestamp,
entry_page: exit_page: event.pathname,
if(session.entry_page == "" and event.name == "pageview",
do: event.pathname,
else: session.entry_page
),
hostname:
if(event.name == "pageview" and session.hostname == "",
do: event.hostname,
else: session.hostname
),
exit_page: if(event.name == "pageview", do: event.pathname, else: session.exit_page),
exit_page_hostname:
if(event.name == "pageview", do: event.hostname, else: session.exit_page_hostname),
is_bounce: false, is_bounce: false,
duration: Timex.diff(event.timestamp, session.start, :second) |> abs, duration: Timex.diff(event.timestamp, session.start, :second) |> abs,
pageviews: pageviews:
@ -67,12 +60,11 @@ defmodule Plausible.Session.CacheStore do
%Plausible.ClickhouseSessionV2{ %Plausible.ClickhouseSessionV2{
sign: 1, sign: 1,
session_id: Plausible.ClickhouseSessionV2.random_uint64(), session_id: Plausible.ClickhouseSessionV2.random_uint64(),
hostname: if(event.name == "pageview", do: event.hostname, else: ""), hostname: event.hostname,
site_id: event.site_id, site_id: event.site_id,
user_id: event.user_id, user_id: event.user_id,
entry_page: if(event.name == "pageview", do: event.pathname, else: ""), entry_page: event.pathname,
exit_page: if(event.name == "pageview", do: event.pathname, else: ""), exit_page: event.pathname,
exit_page_hostname: if(event.name == "pageview", do: event.hostname, else: ""),
is_bounce: true, is_bounce: true,
duration: 0, duration: 0,
pageviews: if(event.name == "pageview", do: 1, else: 0), pageviews: if(event.name == "pageview", do: 1, else: 0),

@ -24,7 +24,7 @@ defmodule Plausible.Shield.CountryRule do
|> cast(attrs, [:site_id, :country_code]) |> cast(attrs, [:site_id, :country_code])
|> validate_required([:site_id, :country_code]) |> validate_required([:site_id, :country_code])
|> validate_length(:country_code, is: 2) |> validate_length(:country_code, is: 2)
|> validate_change(:country_code, fn :country_code, cc -> |> Ecto.Changeset.validate_change(:country_code, fn :country_code, cc ->
if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do if cc in Enum.map(Location.Country.all(), & &1.alpha_2) do
[] []
else else

@ -19,7 +19,7 @@ defmodule Plausible.Shield.CountryRuleCache do
def name(), do: @cache_name def name(), do: @cache_name
@impl true @impl true
def child_id(), do: :cache_country_blocklist def child_id(), do: :cachex_country_blocklist
@impl true @impl true
def count_all() do def count_all() do

@ -19,7 +19,7 @@ defmodule Plausible.Shield.IPRuleCache do
def name(), do: @cache_name def name(), do: @cache_name
@impl true @impl true
def child_id(), do: :cache_ip_blocklist def child_id(), do: :cachex_ip_blocklist
@impl true @impl true
def count_all() do def count_all() do
@ -38,6 +38,8 @@ defmodule Plausible.Shield.IPRuleCache do
end end
@impl true @impl true
def get_from_source({_, "0.0.0.0"}), do: nil
def get_from_source({domain, address}) do def get_from_source({domain, address}) do
query = query =
base_db_query() base_db_query()

@ -1,85 +0,0 @@
defmodule CompiledRegex do
@moduledoc """
Ensures that the regex is compiled on load
"""
use Ecto.Type
def type, do: :string
def cast(val) when is_binary(val), do: {:ok, val}
def cast(_), do: :error
def load(val), do: {:ok, Regex.compile!(val)}
def dump(val), do: {:ok, val}
end
defmodule Plausible.Shield.PageRule do
@moduledoc """
Schema for Pages block list
"""
use Ecto.Schema
import Ecto.Changeset
@type t() :: %__MODULE__{}
@primary_key {:id, :binary_id, autogenerate: true}
schema "shield_rules_page" do
belongs_to :site, Plausible.Site
field :page_path, :string
field :page_path_pattern, CompiledRegex
field :action, Ecto.Enum, values: [:deny, :allow], default: :deny
field :added_by, :string
# If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache`
field :from_cache?, :boolean, virtual: true, default: false
timestamps()
end
def changeset(rule \\ %__MODULE__{}, attrs) do
rule
|> cast(attrs, [:site_id, :page_path])
|> validate_required([:site_id, :page_path])
|> validate_length(:page_path, max: 250)
|> validate_change(:page_path, fn :page_path, p ->
if not String.starts_with?(p, "/") do
[page_path: "must start with /"]
else
[]
end
end)
|> store_regex()
|> unique_constraint(:page_path_pattern,
name: :shield_rules_page_site_id_page_path_pattern_index,
error_key: :page_path,
message: "rule already exists"
)
end
defp store_regex(changeset) do
case get_field(changeset, :page_path) do
"/" <> _ = page_path ->
regex =
page_path
|> Regex.escape()
|> String.replace("\\*\\*", ".*")
|> String.replace("\\*", ".*")
regex = "^#{regex}$"
verify_valid_regex(changeset, regex)
_ ->
changeset
end
end
defp verify_valid_regex(changeset, regex) do
case Regex.compile(regex) do
{:ok, _} ->
put_change(changeset, :page_path_pattern, regex)
{:error, _} ->
add_error(changeset, :page_path, "could not compile regular expression")
end
end
end

@ -1,65 +0,0 @@
defmodule Plausible.Shield.PageRuleCache do
@moduledoc """
Allows retrieving Page Rules by domain
"""
alias Plausible.Shield.PageRule
import Ecto.Query
use Plausible.Cache
@cache_name :page_blocklist_by_domain
@cached_schema_fields ~w(
id
page_path_pattern
action
)a
@impl true
def name(), do: @cache_name
@impl true
def child_id(), do: :cache_page_blocklist
@impl true
def count_all() do
Plausible.Repo.aggregate(PageRule, :count)
end
@impl true
def base_db_query() do
from rule in PageRule,
inner_join: s in assoc(rule, :site),
select: {
s.domain,
s.domain_changed_from,
%{struct(rule, ^@cached_schema_fields) | from_cache?: true}
}
end
@impl true
def get_from_source(domain) do
query =
base_db_query()
|> where([..., site], site.domain == ^domain)
case Plausible.Repo.one(query) do
{_, _, rule} -> %PageRule{rule | from_cache?: false}
_any -> nil
end
end
@impl true
def unwrap_cache_keys(items) do
Enum.reduce(items, [], fn
{domain, nil, object}, acc ->
[{domain, object} | acc]
{domain, domain_changed_from, object}, acc ->
[
{domain, object},
{domain_changed_from, object} | acc
]
end)
end
end

@ -13,63 +13,11 @@ defmodule Plausible.Shields do
@maximum_country_rules 30 @maximum_country_rules 30
def maximum_country_rules(), do: @maximum_country_rules def maximum_country_rules(), do: @maximum_country_rules
@maximum_page_rules 30
def maximum_page_rules(), do: @maximum_page_rules
@spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()]
def list_ip_rules(site_or_id) do def list_ip_rules(site_or_id) do
list(Shield.IPRule, site_or_id) list(Shield.IPRule, site_or_id)
end end
@spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def ip_blocked?(%Site{domain: domain}, address) do
ip_blocked?(domain, address)
end
def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do
case Shield.IPRuleCache.get({domain, address}) do
%Shield.IPRule{action: :deny} ->
true
_ ->
false
end
end
@spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def page_blocked?(%Site{domain: domain}, address) do
page_blocked?(domain, address)
end
def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do
page_rules = Shield.PageRuleCache.get(domain)
if page_rules do
page_rules
|> List.wrap()
|> Enum.find_value(false, fn rule ->
rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname)
end)
else
false
end
end
@spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean()
def country_blocked?(%Site{domain: domain}, country_code) do
country_blocked?(domain, country_code)
end
def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do
case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do
%Shield.CountryRule{action: :deny} ->
true
_ ->
false
end
end
@spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()}
def add_ip_rule(site_or_id, params, opts \\ []) do def add_ip_rule(site_or_id, params, opts \\ []) do
@ -111,28 +59,6 @@ defmodule Plausible.Shields do
count(Shield.CountryRule, site_or_id) count(Shield.CountryRule, site_or_id)
end end
@spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()]
def list_page_rules(site_or_id) do
list(Shield.PageRule, site_or_id)
end
@spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) ::
{:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()}
def add_page_rule(site_or_id, params, opts \\ []) do
opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules})
add(Shield.PageRule, site_or_id, params, opts)
end
@spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok
def remove_page_rule(site_or_id, rule_id) do
remove(Shield.PageRule, site_or_id, rule_id)
end
@spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer()
def count_page_rules(site_or_id) do
count(Shield.PageRule, site_or_id)
end
defp list(schema, %Site{id: id}) do defp list(schema, %Site{id: id}) do
list(schema, id) list(schema, id)
end end

@ -1,136 +0,0 @@
defmodule Plausible.Shields.CountryTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_country_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{})
assert changeset.errors == [country_code: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported country", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "0X"})
assert changeset.errors == [country_code: {"is invalid", []}]
refute changeset.valid?
end
test "incorrect country format", %{site: site} do
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "Germany"})
assert changeset.errors ==
[
{:country_code, {"is invalid", []}},
{:country_code,
{"should be %{count} character(s)",
[count: 2, validation: :length, kind: :is, type: :string]}}
]
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert {:error, changeset} = add_country_rule(site, %{"country_code" => "EE"})
refute changeset.valid?
assert changeset.errors == [
country_code:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_country_site_id_country_code_index"}
]}
]
end
test "over limit", %{site: site} do
country_codes =
Location.Country.all()
|> Enum.take(Plausible.Shields.maximum_country_rules())
|> Enum.map(& &1.alpha_2)
for cc <- country_codes do
assert {:ok, _} =
add_country_rule(site, %{"country_code" => cc})
end
assert count_country_rules(site) == maximum_country_rules()
assert {:error, changeset} =
add_country_rule(site, %{"country_code" => "US"})
refute changeset.valid?
assert changeset.errors == [country_code: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_country_rule(site, %{"country_code" => "EE"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_country_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_country_rule(site, %{"country_code" => "EE"})
assert remove_country_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.CountryRule, rule.id)
assert remove_country_rule(site, rule.id) == :ok
end
end
describe "list_country_rules/1" do
test "empty", %{site: site} do
assert(list_country_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_country_rule(site, %{"country_code" => "EE"})
:timer.sleep(1000)
{:ok, r2} = add_country_rule(site, %{"country_code" => "PL"})
assert [^r2, ^r1] = list_country_rules(site)
end
end
describe "count_country_rules/1" do
test "counts", %{site: site} do
assert count_country_rules(site) == 0
{:ok, _} = add_country_rule(site, %{"country_code" => "EE"})
assert count_country_rules(site) == 1
{:ok, _} = add_country_rule(site, %{"country_code" => "PL"})
assert count_country_rules(site) == 2
end
end
describe "Country Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_country_rules(site.id) == 0
assert list_country_rules(site.id) == []
assert {:ok, rule} =
add_country_rule(site.id, %{"country_code" => "EE"})
add_country_rule(site2, %{"country_code" => "PL"})
assert count_country_rules(site) == 1
assert [^rule] = list_country_rules(site)
assert rule.country_code == "EE"
assert rule.action == :deny
refute rule.from_cache?
assert country_blocked?(site, "ee")
refute country_blocked?(site, "xx")
end
end
end

@ -1,147 +0,0 @@
defmodule Plausible.Shields.IPTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_ip_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{})
assert changeset.errors == [inet: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "unsupported netmask", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "127.0.0.0/24"})
assert changeset.errors == [inet: {"netmask unsupported", []}]
refute changeset.valid?
assert {:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.0/32"})
end
test "incorrect ip", %{site: site} do
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "999.999.999.999"})
assert changeset.errors == [
inet: {"is invalid", [{:type, EctoNetwork.INET}, {:validation, :cast}]}
]
refute changeset.valid?
end
test "non-strict IPs", %{site: site} do
assert {:error, _} = add_ip_rule(site, %{"inet" => "111"})
end
test "double insert", %{site: site} do
assert {:ok, _} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
assert {:error, changeset} = add_ip_rule(site, %{"inet" => "0.0.0.111"})
refute changeset.valid?
assert changeset.errors == [
inet:
{"has already been taken",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_ip_site_id_inet_index"}
]}
]
end
test "ipv6", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "2001:0000:130F:0000:0000:09C0:876A:130B"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "ipv4", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"})
assert ^rule = Repo.get(Plausible.Shield.IPRule, rule.id)
end
test "over limit", %{site: site} do
for i <- 1..maximum_ip_rules() do
assert {:ok, _} =
add_ip_rule(site, %{"inet" => "1.1.1.#{i}"})
end
assert count_ip_rules(site) == maximum_ip_rules()
assert {:error, changeset} =
add_ip_rule(site, %{"inet" => "1.1.1.31"})
refute changeset.valid?
assert changeset.errors == [inet: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_ip_rule(site, %{"inet" => "1.1.1.1"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "remove_ip_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert remove_ip_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.IPRule, rule.id)
assert remove_ip_rule(site, rule.id) == :ok
end
end
describe "list_ip_rules/1" do
test "empty", %{site: site} do
assert(list_ip_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
:timer.sleep(1000)
{:ok, r2} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert [^r2, ^r1] = list_ip_rules(site)
end
end
describe "count_ip_rules/1" do
test "counts", %{site: site} do
assert count_ip_rules(site) == 0
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.1"})
assert count_ip_rules(site) == 1
{:ok, _} = add_ip_rule(site, %{"inet" => "127.0.0.2"})
assert count_ip_rules(site) == 2
end
end
describe "IP Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_ip_rules(site.id) == 0
assert list_ip_rules(site.id) == []
assert {:ok, rule} =
add_ip_rule(site.id, %{"inet" => "127.0.0.1", "description" => "Localhost"})
add_ip_rule(site2, %{"inet" => "127.0.0.1", "description" => "Localhost"})
assert count_ip_rules(site) == 1
assert [^rule] = list_ip_rules(site)
assert rule.inet == %Postgrex.INET{address: {127, 0, 0, 1}, netmask: 32}
assert rule.description == "Localhost"
assert rule.action == :deny
refute rule.from_cache?
assert ip_blocked?(site, "127.0.0.1")
refute ip_blocked?(site, "127.0.0.2")
end
end
end

@ -1,174 +0,0 @@
defmodule Plausible.Shields.PageTest do
use Plausible.DataCase
import Plausible.Shields
setup do
site = insert(:site)
{:ok, %{site: site}}
end
describe "add_page_rule/2" do
test "no input", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{})
assert changeset.errors == [page_path: {"can't be blank", [validation: :required]}]
refute changeset.valid?
end
test "no slash", %{site: site} do
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "test"})
assert changeset.errors == [page_path: {"must start with /", []}]
refute changeset.valid?
end
test "lengthy", %{site: site} do
long = "/" <> :binary.copy("a", 251)
assert {:error, changeset} = add_page_rule(site, %{"page_path" => long})
assert [page_path: {"should be at most %{count} character(s)", _}] = changeset.errors
refute changeset.valid?
end
test "double insert", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test"})
refute changeset.valid?
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "equivalent rules are counted as dupes", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
assert {:error, changeset} = add_page_rule(site, %{"page_path" => "/test/**"})
assert changeset.errors == [
page_path:
{"rule already exists",
[
{:constraint, :unique},
{:constraint_name, "shield_rules_page_site_id_page_path_pattern_index"}
]}
]
end
test "regex storage: wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*"})
assert rule.page_path_pattern == "^/test/.*$"
end
test "regex storage: no wildcard", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert rule.page_path_pattern == "^/test$"
end
test "regex storage: escaping", %{site: site} do
assert {:ok, rule} = add_page_rule(site, %{"page_path" => "/test/*/**/|+[0-9]"})
assert rule.page_path_pattern == "^/test/.*/.*/\\|\\+\\[0\\-9\\]$"
end
test "over limit", %{site: site} do
for i <- 1..maximum_page_rules() do
assert {:ok, _} =
add_page_rule(site, %{"page_path" => "/test/#{i}"})
end
assert count_page_rules(site) == maximum_page_rules()
assert {:error, changeset} =
add_page_rule(site, %{"page_path" => "/test/31"})
refute changeset.valid?
assert changeset.errors == [page_path: {"maximum reached", []}]
end
test "with added_by", %{site: site} do
assert {:ok, rule} =
add_page_rule(site, %{"page_path" => "/test"},
added_by: build(:user, name: "Joe", email: "joe@example.com")
)
assert rule.added_by == "Joe <joe@example.com>"
end
end
describe "page pattern matching" do
test "no wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test"})
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
end
test "wildcard", %{site: site} do
assert {:ok, _} = add_page_rule(site, %{"page_path" => "/test/*"})
refute page_blocked?(site, "/test")
assert page_blocked?(site, "/test/")
assert page_blocked?(site, "/test/hello")
refute page_blocked?(site, "test")
refute page_blocked?(site, "/testing")
end
end
describe "remove_page_rule/2" do
test "is idempontent", %{site: site} do
{:ok, rule} = add_page_rule(site, %{"page_path" => "/test"})
assert remove_page_rule(site, rule.id) == :ok
refute Repo.get(Plausible.Shield.PageRule, rule.id)
assert remove_page_rule(site, rule.id) == :ok
end
end
describe "list_page_rules/1" do
test "empty", %{site: site} do
assert(list_page_rules(site) == [])
end
@tag :slow
test "many", %{site: site} do
{:ok, r1} = add_page_rule(site, %{"page_path" => "/test"})
:timer.sleep(1000)
{:ok, r2} = add_page_rule(site, %{"page_path" => "/test"})
assert [^r2, ^r1] = list_page_rules(site)
end
end
describe "count_page_rules/1" do
test "counts", %{site: site} do
assert count_page_rules(site) == 0
{:ok, _} = add_page_rule(site, %{"page_path" => "/test1"})
assert count_page_rules(site) == 1
{:ok, _} = add_page_rule(site, %{"page_path" => "/test2"})
assert count_page_rules(site) == 2
end
end
describe "Page Rules" do
test "end to end", %{site: site} do
site2 = insert(:site)
assert count_page_rules(site.id) == 0
assert list_page_rules(site.id) == []
assert {:ok, rule} =
add_page_rule(site.id, %{
"page_path" => "/test"
})
add_page_rule(site2, %{"page_path" => "/test"})
assert count_page_rules(site) == 1
assert [%{id: rule_id}] = list_page_rules(site)
assert rule.id == rule_id
assert rule.page_path == "/test"
assert rule.action == :deny
refute rule.from_cache?
assert page_blocked?(site, "/test")
refute page_blocked?(site, "/testing")
end
end
end

@ -59,7 +59,7 @@ defmodule Plausible.Site do
# `site` in `assigns`. # `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true field :earliest_import_start_date, :date, virtual: true
field :latest_import_end_date, :date, virtual: true field :earliest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps() timestamps()

@ -37,7 +37,7 @@ defmodule Plausible.Site.Cache do
def name(), do: @cache_name def name(), do: @cache_name
@impl true @impl true
def child_id(), do: :cache_sites def child_id(), do: :cachex_sites
@impl true @impl true
def count_all() do def count_all() do

@ -23,14 +23,6 @@ defmodule Plausible.Sites do
Repo.get_by!(Site, domain: domain) Repo.get_by!(Site, domain: domain)
end end
def get_domain!(site_id) do
Plausible.Repo.one!(
from s in Plausible.Site,
where: [id: ^site_id],
select: s.domain
)
end
@spec toggle_pin(Auth.User.t(), Site.t()) :: @spec toggle_pin(Auth.User.t(), Site.t()) ::
{:ok, Site.UserPreference.t()} | {:error, :too_many_pins} {:ok, Site.UserPreference.t()} | {:error, :too_many_pins}
def toggle_pin(user, site) do def toggle_pin(user, site) do

@ -5,6 +5,17 @@ defmodule Plausible.Stats.Aggregate do
import Ecto.Query import Ecto.Query
alias Plausible.Stats.{Query, Util} alias Plausible.Stats.{Query, Util}
@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])
@event_metrics [
:visitors,
:pageviews,
:events,
:sample_percent
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
def aggregate(site, query, metrics) do def aggregate(site, query, metrics) do
{currency, metrics} = {currency, metrics} =
on_full_build do on_full_build do
@ -15,17 +26,18 @@ defmodule Plausible.Stats.Aggregate do
Query.trace(query, metrics) Query.trace(query, metrics)
{event_metrics, session_metrics, other_metrics} = event_metrics =
metrics metrics
|> Util.maybe_add_visitors_metric() |> Util.maybe_add_visitors_metric()
|> Plausible.Stats.TableDecider.partition_metrics(query) |> Enum.filter(&(&1 in @event_metrics))
event_task = fn -> aggregate_events(site, query, event_metrics) end event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
session_task = fn -> aggregate_sessions(site, query, session_metrics) end session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task = time_on_page_task =
if :time_on_page in other_metrics do if :time_on_page in metrics do
fn -> aggregate_time_on_page(site, query) end fn -> aggregate_time_on_page(site, query) end
else else
fn -> %{} end fn -> %{} end
@ -33,6 +45,7 @@ defmodule Plausible.Stats.Aggregate do
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task]) Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end) |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> maybe_put_cr(site, query, metrics)
|> Util.keep_requested_metrics(metrics) |> Util.keep_requested_metrics(metrics)
|> cast_revenue_metrics_to_money(currency) |> cast_revenue_metrics_to_money(currency)
|> Enum.map(&maybe_round_value/1) |> Enum.map(&maybe_round_value/1)
@ -40,20 +53,45 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{}) |> Enum.into(%{})
end end
defp maybe_put_cr(aggregate_result, site, query, metrics) do
if :conversion_rate in metrics do
all =
query
|> Query.remove_event_filters([:goal, :props])
|> then(fn query -> aggregate_events(site, query, [:visitors]) end)
|> Map.fetch!(:visitors)
converted = aggregate_result.visitors
cr = Util.calculate_cr(all, converted)
aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)
if :total_visitors in metrics do
Map.put(aggregate_result, :total_visitors, all)
else
aggregate_result
end
else
aggregate_result
end
end
defp aggregate_events(_, _, []), do: %{} defp aggregate_events(_, _, []), do: %{}
defp aggregate_events(site, query, metrics) do defp aggregate_events(site, query, metrics) do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics)) from(e in base_event_query(site, query), select: %{})
|> select_event_metrics(metrics)
|> merge_imported(site, query, :aggregate, metrics) |> merge_imported(site, query, :aggregate, metrics)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
|> ClickhouseRepo.one() |> ClickhouseRepo.one()
end end
defp aggregate_sessions(_, _, []), do: %{} defp aggregate_sessions(_, _, []), do: %{}
defp aggregate_sessions(site, query, metrics) do defp aggregate_sessions(site, query, metrics) do
from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query)) from(e in query_sessions(site, query), select: %{})
|> filter_converted_sessions(site, query) |> filter_converted_sessions(site, query)
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, :aggregate, metrics) |> merge_imported(site, query, :aggregate, metrics)
|> ClickhouseRepo.one() |> ClickhouseRepo.one()
|> Util.keep_requested_metrics(metrics) |> Util.keep_requested_metrics(metrics)
@ -192,7 +230,7 @@ defmodule Plausible.Stats.Aggregate do
@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent] @metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
defp maybe_round_value({metric, nil}), do: {metric, nil} defp maybe_round_value({metric, nil}), do: {metric, 0}
defp maybe_round_value({metric, value}) when metric in @metrics_to_round do defp maybe_round_value({metric, value}) when metric in @metrics_to_round do
{metric, round(value)} {metric, round(value)}

@ -3,7 +3,7 @@ defmodule Plausible.Stats.Base do
use Plausible use Plausible
use Plausible.Stats.Fragments use Plausible.Stats.Fragments
alias Plausible.Stats.{Query, Filters, TableDecider} alias Plausible.Stats.{Query, Filters}
alias Plausible.Timezones alias Plausible.Timezones
import Ecto.Query import Ecto.Query
@ -15,7 +15,7 @@ defmodule Plausible.Stats.Base do
def base_event_query(site, query) do def base_event_query(site, query) do
events_q = query_events(site, query) events_q = query_events(site, query)
if TableDecider.events_join_sessions?(query) do if Enum.any?(Filters.visit_props(), &query.filters["visit:" <> &1]) do
sessions_q = sessions_q =
from( from(
s in query_sessions(site, query), s in query_sessions(site, query),
@ -30,15 +30,11 @@ defmodule Plausible.Stats.Base do
on: e.session_id == sq.session_id on: e.session_id == sq.session_id
) )
else else
if query.experimental_reduced_joins? do events_q
events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)
else
events_q
end
end end
end end
defp query_events(site, query) do def query_events(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site) {first_datetime, last_datetime} = utc_boundaries(query, site)
q = q =
@ -52,10 +48,7 @@ defmodule Plausible.Stats.Base do
q = Plausible.Stats.Sampling.add_query_hint(q, query) q = Plausible.Stats.Sampling.add_query_hint(q, query)
end end
q = q = from(e in q, where: ^dynamic_filter_condition(query, "event:page", :pathname))
q
|> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))
|> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))
q = q =
case query.filters["event:name"] do case query.filters["event:name"] do
@ -129,6 +122,17 @@ defmodule Plausible.Stats.Base do
q q
end end
@api_prop_name_to_db %{
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id"
}
def query_sessions(site, query) do def query_sessions(site, query) do
{first_datetime, last_datetime} = {first_datetime, last_datetime} =
utc_boundaries(query, site) utc_boundaries(query, site)
@ -136,7 +140,8 @@ defmodule Plausible.Stats.Base do
q = from(s in "sessions_v2", where: s.site_id == ^site.id) q = from(s in "sessions_v2", where: s.site_id == ^site.id)
sessions_q = sessions_q =
if query.experimental_session_count? do if FunWithFlags.enabled?(:experimental_session_count, for: site) or
query.experimental_session_count? do
from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
else else
from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime
@ -146,24 +151,9 @@ defmodule Plausible.Stats.Base do
sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query) sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
end end
filter_by_entry_props(sessions_q, query) sessions_q = filter_by_entry_props(sessions_q, query)
|> filter_by_visit_props(Filters.visit_props(), query)
end
@api_prop_name_to_db %{ Enum.reduce(Filters.visit_props(), sessions_q, fn prop_name, sessions_q ->
"source" => "referrer_source",
"device" => "screen_size",
"screen" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code",
"region" => "subdivision1_code",
"city" => "city_geoname_id",
"entry_page_hostname" => "hostname"
}
defp filter_by_visit_props(q, visit_props, query) do
Enum.reduce(visit_props, q, fn prop_name, sessions_q ->
filter_key = "visit:" <> prop_name filter_key = "visit:" <> prop_name
db_field = db_field =
@ -219,159 +209,163 @@ defmodule Plausible.Stats.Base do
def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q
def select_event_metrics(metrics) do def select_event_metrics(q, []), do: q
metrics
|> Enum.map(&select_event_metric/1)
|> Enum.reduce(%{}, &Map.merge/2)
end
defp select_event_metric(:pageviews) do def select_event_metrics(q, [:pageviews | rest]) do
%{ from(e in q,
pageviews: select_merge: %{
dynamic( pageviews:
[e],
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
) }
} )
end |> select_event_metrics(rest)
defp select_event_metric(:events) do
%{
events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
}
end end
defp select_event_metric(:visitors) do def select_event_metrics(q, [:events | rest]) do
%{ from(e in q,
visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)) select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")}
} )
|> select_event_metrics(rest)
end end
defp select_event_metric(:visits) do def select_event_metrics(q, [:visitors | rest]) do
%{ from(e in q,
visits: select_merge: %{
dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)) visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)
} }
)
|> select_event_metrics(rest)
end end
on_full_build do on_full_build do
defp select_event_metric(:total_revenue) do def select_event_metrics(q, [:total_revenue | rest]) do
%{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()} q
|> Plausible.Stats.Goal.Revenue.total_revenue_query()
|> select_event_metrics(rest)
end end
defp select_event_metric(:average_revenue) do def select_event_metrics(q, [:average_revenue | rest]) do
%{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()} q
|> Plausible.Stats.Goal.Revenue.average_revenue_query()
|> select_event_metrics(rest)
end end
end end
defp select_event_metric(:sample_percent) do def select_event_metrics(q, [:sample_percent | rest]) do
%{ from(e in q,
sample_percent: select_merge: %{
dynamic( sample_percent:
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
) }
} )
|> select_event_metrics(rest)
end end
defp select_event_metric(:percentage), do: %{} def select_event_metrics(q, [:percentage | rest]) do
defp select_event_metric(:conversion_rate), do: %{} q |> select_event_metrics(rest)
defp select_event_metric(:total_visitors), do: %{} end
defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}") def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}")
def select_session_metrics(metrics, query) do def select_session_metrics(q, [], _query), do: q
metrics
|> Enum.map(&select_session_metric(&1, query))
|> Enum.reduce(%{}, &Map.merge/2)
end
defp select_session_metric(:bounce_rate, query) do def select_session_metrics(q, [:bounce_rate | rest], query) do
condition = dynamic_filter_condition(query, "event:page", :entry_page) condition = dynamic_filter_condition(query, "event:page", :entry_page)
%{ from(s in q,
bounce_rate: select_merge:
dynamic( ^%{
[], bounce_rate:
fragment( dynamic(
"toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))", [],
^condition, fragment(
^condition "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
) ^condition,
), ^condition
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) )
} ),
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
}
)
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:visits, _query) do def select_session_metrics(q, [:visits | rest], query) do
%{ from(s in q,
visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)) select_merge: %{
} visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)
}
)
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:pageviews, _query) do def select_session_metrics(q, [:pageviews | rest], query) do
%{ from(s in q,
pageviews: select_merge: %{
dynamic( pageviews:
[s],
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
) }
} )
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:events, _query) do def select_session_metrics(q, [:events | rest], query) do
%{ from(s in q,
events: select_merge: %{
dynamic( events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
[s], }
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events) )
) |> select_session_metrics(rest, query)
}
end end
defp select_session_metric(:visitors, _query) do def select_session_metrics(q, [:visitors | rest], query) do
%{ from(s in q,
visitors: select_merge: %{
dynamic( visitors:
[s],
selected_as( selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id), fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),
:visitors :visitors
) )
) }
} )
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:visit_duration, _query) do def select_session_metrics(q, [:visit_duration | rest], query) do
%{ from(s in q,
visit_duration: select_merge: %{
dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")), :visit_duration =>
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
} __internal_visits: fragment("toUInt32(sum(sign))")
}
)
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:views_per_visit, _query) do def select_session_metrics(q, [:views_per_visit | rest], query) do
%{ from(s in q,
views_per_visit: select_merge: %{
dynamic( views_per_visit:
[s], fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign),
fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign) __internal_visits: fragment("toUInt32(sum(sign))")
), }
__internal_visits: dynamic([], fragment("toUInt32(sum(sign))")) )
} |> select_session_metrics(rest, query)
end end
defp select_session_metric(:sample_percent, _query) do def select_session_metrics(q, [:sample_percent | rest], query) do
%{ from(e in q,
sample_percent: select_merge: %{
dynamic( sample_percent:
[],
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
) }
} )
|> select_session_metrics(rest, query)
end end
defp select_session_metric(:percentage, _query), do: %{} def select_session_metrics(q, [:percentage | rest], query) do
q |> select_session_metrics(rest, query)
end
def dynamic_filter_condition(query, filter_key, db_field) do def dynamic_filter_condition(query, filter_key, db_field) do
case query && query.filters && query.filters[filter_key] do case query && query.filters && query.filters[filter_key] do
@ -592,7 +586,7 @@ defmodule Plausible.Stats.Base do
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
end end
defp total_visitors_subquery(site, query, true) do defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
dynamic( dynamic(
[e], [e],
selected_as( selected_as(
@ -603,16 +597,14 @@ defmodule Plausible.Stats.Base do
) )
end end
defp total_visitors_subquery(site, query, false) do defp total_visitors_subquery(site, query) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors)) dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end end
def add_percentage_metric(q, site, query, metrics) do def add_percentage_metric(q, site, query, metrics) do
if :percentage in metrics do if :percentage in metrics do
q q
|> select_merge( |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}
)
|> select_merge(%{ |> select_merge(%{
percentage: percentage:
fragment( fragment(
@ -626,32 +618,4 @@ defmodule Plausible.Stats.Base do
q q
end end
end end
# Adds conversion_rate metric to query, calculated as
# X / Y where Y is the same breakdown value without goal or props
# filters.
def maybe_add_conversion_rate(q, site, query, metrics, opts) do
if :conversion_rate in metrics do
include_imported = Keyword.fetch!(opts, :include_imported)
total_query = query |> Query.remove_event_filters([:goal, :props])
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
|> select_merge(
^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}
)
|> select_merge([e], %{
conversion_rate:
fragment(
"if(? > 0, round(? / ? * 100, 1), 0)",
selected_as(:__total_visitors),
e.visitors,
selected_as(:__total_visitors)
)
})
else
q
end
end
end end

@ -4,9 +4,8 @@ defmodule Plausible.Stats.Breakdown do
use Plausible.Stats.Fragments use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported} import Plausible.Stats.{Base, Imported}
import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util, TableDecider} alias Plausible.Stats.{Query, Util}
@no_ref "Direct / None" @no_ref "Direct / None"
@not_set "(not set)" @not_set "(not set)"
@ -53,71 +52,50 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
event_q = event_results =
if Enum.any?(event_goals) do if Enum.any?(event_goals) do
site site
|> breakdown_events(event_query, "event:name", metrics_to_select) |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)
|> apply_pagination(pagination) |> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
else else
nil []
end end
page_q = {limit, page} = pagination
offset = (page - 1) * limit
page_results =
if Enum.any?(pageview_goals) do if Enum.any?(pageview_goals) do
page_exprs = Enum.map(pageview_goals, & &1.page_path) page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1) page_regexes = Enum.map(page_exprs, &page_regex/1)
select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
from(e in base_event_query(site, query), from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)], order_by: [desc: fragment("uniq(?)", e.user_id)],
limit: ^limit,
offset: ^offset,
where: where:
fragment( fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)", "notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname, e.pathname,
^page_regexes ^page_regexes
) and e.name == "pageview", ) and e.name == "pageview",
array_join: index in fragment("indices"), group_by: fragment("index"),
group_by: index,
select: %{ select: %{
name: fragment("concat('Visit ', ?[?])", ^page_exprs, index) index: fragment("arrayJoin(indices) as index"),
goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
} }
) )
|> select_merge(^select_columns) |> select_event_metrics(metrics_to_select -- @revenue_metrics)
|> apply_pagination(pagination) |> ClickhouseRepo.all()
|> Enum.map(fn row -> Map.delete(row, :index) end)
else else
nil []
end
full_q =
case {event_q, page_q} do
{nil, nil} ->
nil
{event_q, nil} ->
event_q
{nil, page_q} ->
page_q
{event_q, page_q} ->
from(
e in subquery(union_all(event_q, ^page_q)),
order_by: [desc: e.visitors]
)
|> apply_pagination(pagination)
end end
if full_q do zip_results(event_results, page_results, :goal, metrics_to_select)
full_q |> maybe_add_cr(site, query, nil, metrics)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: false) |> Util.keep_requested_metrics(metrics)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
|> Util.keep_requested_metrics(metrics)
else
[]
end
end end
def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do
@ -132,11 +110,11 @@ defmodule Plausible.Stats.Breakdown do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select) breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)
|> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{breakdown_prop_value: custom_prop})
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency)) |> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
|> Enum.sort_by(& &1[sorting_key(metrics_to_select)], :desc)
|> maybe_add_cr(site, query, nil, metrics)
|> Util.keep_requested_metrics(metrics)
end end
def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do
@ -147,10 +125,10 @@ defmodule Plausible.Stats.Breakdown do
event_result = event_result =
site site
|> breakdown_events(query, property, event_metrics) |> breakdown_events(query, "event:page", event_metrics, pagination)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics) |> maybe_add_time_on_page(site, query, metrics)
|> maybe_add_cr(site, query, property, metrics, pagination)
|> Util.keep_requested_metrics(metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
@ -171,8 +149,7 @@ defmodule Plausible.Stats.Breakdown do
{limit, _page} = pagination {limit, _page} = pagination
session_result = session_result =
breakdown_sessions(site, new_query, "visit:entry_page", session_metrics) breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})
|> paginate_and_execute(session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page}) |> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page] metrics = metrics ++ [:page]
@ -189,9 +166,7 @@ defmodule Plausible.Stats.Breakdown do
def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics) if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
breakdown_events(site, query, property, metrics, pagination)
breakdown_events(site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end end
def breakdown(site, query, property, metrics, pagination, opts) do def breakdown(site, query, property, metrics, pagination, opts) do
@ -199,35 +174,9 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
case breakdown_table(query, metrics, property) do breakdown_sessions(site, query, property, metrics_to_select, pagination)
:session -> |> maybe_add_cr(site, query, property, metrics, pagination)
breakdown_sessions(site, query, property, metrics_to_select) |> Util.keep_requested_metrics(metrics)
|> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
:event ->
breakdown_events(site, query, property, metrics_to_select)
|> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
|> paginate_and_execute(metrics, pagination)
end
end
# Backwards compatibility
# defp breakdown_table(%Query{experimental_reduced_joins?: false}, _, _), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:entry_page_hostname"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page"), do: :session
defp breakdown_table(_query, _metrics, "visit:exit_page_hostname"), do: :session
defp breakdown_table(query, metrics, property) do
{_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, property)
if not Enum.empty?(session_metrics) do
:session
else
:event
end
end end
defp zip_results(event_result, session_result, property, metrics) do defp zip_results(event_result, session_result, property, metrics) do
@ -245,35 +194,38 @@ defmodule Plausible.Stats.Breakdown do
|> Map.merge(event_row) |> Map.merge(event_row)
|> Map.merge(session_row) |> Map.merge(session_row)
end) end)
|> sort_results(metrics) |> Enum.sort_by(& &1[sorting_key(metrics)], :desc)
end end
defp breakdown_sessions(site, query, property, metrics) do defp breakdown_sessions(_, _, _, [], _), do: []
defp breakdown_sessions(site, query, property, metrics, pagination) do
from(s in query_sessions(site, query), from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)], order_by: [desc: fragment("uniq(?)", s.user_id)],
select: ^select_session_metrics(metrics, query) select: %{}
) )
|> filter_converted_sessions(site, query) |> filter_converted_sessions(site, query)
|> do_group_by(property) |> do_group_by(property)
|> select_session_metrics(metrics, query)
|> merge_imported(site, query, property, metrics) |> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics) |> add_percentage_metric(site, query, metrics)
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
|> Util.keep_requested_metrics(metrics)
end end
defp breakdown_events(site, query, property, metrics) do defp breakdown_events(_, _, _, [], _), do: []
defp breakdown_events(site, query, property, metrics, pagination) do
from(e in base_event_query(site, query), from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)], order_by: [desc: fragment("uniq(?)", e.user_id)],
select: %{} select: %{}
) )
|> do_group_by(property) |> do_group_by(property)
|> select_merge(^select_event_metrics(metrics)) |> select_event_metrics(metrics)
|> merge_imported(site, query, property, metrics) |> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics) |> add_percentage_metric(site, query, metrics)
end
defp paginate_and_execute(_, [], _), do: []
defp paginate_and_execute(q, metrics, pagination) do
q
|> apply_pagination(pagination) |> apply_pagination(pagination)
|> ClickhouseRepo.all() |> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os}) |> transform_keys(%{operating_system: :os})
@ -298,6 +250,94 @@ defmodule Plausible.Stats.Breakdown do
end end
defp breakdown_time_on_page(site, query, pages) do defp breakdown_time_on_page(site, query, pages) do
if FunWithFlags.enabled?(:window_time_on_page) do
window_breakdown_time_on_page(site, query, pages)
else
neighbor_breakdown_time_on_page(site, query, pages)
end
end
defp neighbor_breakdown_time_on_page(site, query, pages) do
q =
from(
e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
select: {
fragment("? as p", e.pathname),
fragment("? as t", e.timestamp),
fragment("? as s", e.session_id)
},
order_by: [e.session_id, e.timestamp]
)
{base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q)
select =
if query.include_imported do
"sum(td), count(case when p2 != p then 1 end)"
else
"round(sum(td)/count(case when p2 != p then 1 end))"
end
pages_idx = length(base_query_raw_params)
params = base_query_raw_params ++ [pages]
time_query = "
SELECT
p,
#{select}
FROM
(SELECT
p,
p2,
sum(t2-t) as td
FROM
(SELECT
*,
neighbor(t, 1) as t2,
neighbor(p, 1) as p2,
neighbor(s, 1) as s2
FROM (#{base_query_raw}))
WHERE s=s2 AND p IN {$#{pages_idx}:Array(String)}
GROUP BY p,p2,s)
GROUP BY p"
{:ok, res} = ClickhouseRepo.query(time_query, params)
if query.include_imported do
# Imported page views have pre-calculated values
res =
res.rows
|> Enum.map(fn [page, time, visits] -> {page, {time, visits}} end)
|> Enum.into(%{})
from(
i in "imported_pages",
group_by: i.page,
where: i.site_id == ^site.id,
where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
where: i.page in ^pages,
select: %{
page: i.page,
pageviews: fragment("sum(?) - sum(?)", i.pageviews, i.exits),
time_on_page: sum(i.time_on_page)
}
)
|> ClickhouseRepo.all()
|> Enum.reduce(res, fn %{page: page, pageviews: pageviews, time_on_page: time}, res ->
{restime, resviews} = Map.get(res, page, {0, 0})
Map.put(res, page, {restime + time, resviews + pageviews})
end)
|> Enum.map(fn
{page, {_, 0}} -> {page, nil}
{page, {time, pageviews}} -> {page, time / pageviews}
end)
|> Enum.into(%{})
else
res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{})
end
end
defp window_breakdown_time_on_page(site, query, pages) do
import Ecto.Query import Ecto.Query
windowed_pages_q = windowed_pages_q =
@ -383,7 +423,7 @@ defmodule Plausible.Stats.Breakdown do
from( from(
e in q, e in q,
select_merge: %{ select_merge: %{
breakdown_prop_value: ^prop =>
selected_as( selected_as(
fragment( fragment(
"if(not empty(?), ?, '(none)')", "if(not empty(?), ?, '(none)')",
@ -466,8 +506,6 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:entry_page") do defp do_group_by(q, "visit:entry_page") do
from( from(
s in q, s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.entry_page, group_by: s.entry_page,
select_merge: %{entry_page: s.entry_page}, select_merge: %{entry_page: s.entry_page},
order_by: {:asc, s.entry_page} order_by: {:asc, s.entry_page}
@ -477,8 +515,6 @@ defmodule Plausible.Stats.Breakdown do
defp do_group_by(q, "visit:exit_page") do defp do_group_by(q, "visit:exit_page") do
from( from(
s in q, s in q,
# Sessions without pageviews don't get entry_page assigned, hence they should get ignored
where: s.entry_page != "",
group_by: s.exit_page, group_by: s.exit_page,
select_merge: %{exit_page: s.exit_page}, select_merge: %{exit_page: s.exit_page},
order_by: {:asc, s.exit_page} order_by: {:asc, s.exit_page}
@ -616,42 +652,17 @@ defmodule Plausible.Stats.Breakdown do
) )
end end
defp group_by_field_names("event:props:" <> _prop), do: [:name] defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do
defp group_by_field_names("visit:os"), do: [:operating_system] cond do
defp group_by_field_names("visit:os_version"), do: [:os, :os_version] :conversion_rate not in metrics -> breakdown_results
defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version] Enum.empty?(breakdown_results) -> breakdown_results
is_nil(property) -> add_absolute_cr(breakdown_results, site, query)
defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)] true -> add_cr(breakdown_results, site, query, property, metrics, pagination)
end
defp on_matches_group_by(fields) do
Enum.reduce(fields, nil, &fields_equal/2)
end
defp outer_order_by(fields) do
Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
end
defp fields_equal(field_name, nil),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
defp fields_equal(field_name, condition),
do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
defp sort_results(results, metrics) do
Enum.sort_by(
results,
fn entry ->
case entry[sorting_key(metrics)] do
nil -> 0
n -> n
end
end,
:desc
)
end end
# This function injects a conversion_rate metric into # This function injects a conversion_rate metric into every
# a breakdown query. It is calculated as X / Y, where: # breakdown result map. It is calculated as X / Y, where:
# #
# * X is the number of conversions for a breakdown # * X is the number of conversions for a breakdown
# result (conversion = number of visitors who # result (conversion = number of visitors who
@ -661,41 +672,109 @@ defmodule Plausible.Stats.Breakdown do
# * Y is the number of all visitors for this breakdown # * Y is the number of all visitors for this breakdown
# result without the `event:goal` and `event:props:*` # result without the `event:goal` and `event:props:*`
# filters. # filters.
defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do defp add_cr(breakdown_results, site, query, property, metrics, pagination) do
if :conversion_rate in metrics do property_atom = Plausible.Stats.Filters.without_prefix(property)
breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])
items =
breakdown_total_visitors_q = Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)
breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors])
query_without_goal =
from(e in subquery(q), query
left_join: c in subquery(breakdown_total_visitors_q), |> Query.put_filter(property, {:member, items})
on: ^on_matches_group_by(group_by_field_names(property)), |> Query.remove_event_filters([:goal, :props])
select_merge: %{
total_visitors: c.visitors, # Here, we're always only interested in the first page of results
conversion_rate: # - the :member filter makes sure that the results always match with
fragment( # the items in the given breakdown_results list
"if(? > 0, round(? / ? * 100, 1), 0)", page = 1
c.visitors,
e.visitors, # For browser/os versions we need to fetch a lot more entries than the
c.visitors # pagination limit. This is because many entries can correspond to a
) # single version number and we need to make sure that the results
}, # without goal filter will include all those combinations of browsers/os-s
order_by: [desc: e.visitors], # and their versions that were present in the `breakdown_results` array.
order_by: ^outer_order_by(group_by_field_names(property)) {pagination_limit, find_match_fn} =
) case property_atom do
else :browser_version ->
q pagination_limit = min(elem(pagination, 0) * 10, 3_000)
end
find_match_fn = fn total, conversion ->
total[:browser_version] == conversion[:browser_version] &&
total[:browser] == conversion[:browser]
end
{pagination_limit, find_match_fn}
:os_version ->
pagination_limit = min(elem(pagination, 0) * 5, 3_000)
find_match_fn = fn total, conversion ->
total[:os_version] == conversion[:os_version] &&
total[:os] == conversion[:os]
end
{pagination_limit, find_match_fn}
_ ->
{elem(pagination, 0),
fn total, conversion ->
total[property_atom] == conversion[property_atom]
end}
end
pagination = {pagination_limit, page}
res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)
Enum.map(breakdown_results, fn item ->
without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))
{conversion_rate, total_visitors} =
if without_goal do
{Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}
else
Sentry.capture_message(
"Unable to find a conversion_rate divisor from a breakdown response",
extra: %{
domain: site.domain,
query: inspect(query),
property: property,
pagination: inspect(pagination),
item_not_found: inspect(item)
}
)
{"N/A", "N/A"}
end
if :total_visitors in metrics do
item
|> Map.put(:conversion_rate, conversion_rate)
|> Map.put(:total_visitors, total_visitors)
else
Map.put(item, :conversion_rate, conversion_rate)
end
end)
end end
# When querying custom event goals and pageviewgoals together, UNION ALL is used # Similar to `add_cr/5`, injects a conversion_rate metric into
# so the same fields must be present on both sides of the union. This change to the # every breakdown result. However, a single divisor is used in
# query will ensure that we don't unnecessarily read revenue column for pageview goals # the CR calculation across all breakdown results. That is the
defp mark_revenue_as_nil(select_columns) do # number of visitors without `event:goal` and `event:props:*`
select_columns # filters.
|> Map.replace(:total_revenue, nil) #
|> Map.replace(:average_revenue, nil) # This is useful when we're only interested in the conversions
# themselves - not how well a certain property such as browser
# or page converted.
defp add_absolute_cr(breakdown_results, site, query) do
total_q = Query.remove_event_filters(query, [:goal, :props])
%{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])
breakdown_results
|> Enum.map(fn goal ->
Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))
end)
end end
defp sorting_key(metrics) do defp sorting_key(metrics) do
@ -715,8 +794,8 @@ defmodule Plausible.Stats.Breakdown do
offset = (page - 1) * limit offset = (page - 1) * limit
q q
|> limit(^limit) |> Ecto.Query.limit(^limit)
|> offset(^offset) |> Ecto.Query.offset(^offset)
end end
defp trace(query, property, metrics) do defp trace(query, property, metrics) do

@ -12,7 +12,6 @@ defmodule Plausible.Stats.Compare do
end end
def percent_change(nil, _new_count), do: nil def percent_change(nil, _new_count), do: nil
def percent_change(_old_count, nil), do: nil
def percent_change(%Money{} = old_count, %Money{} = new_count) do def percent_change(%Money{} = old_count, %Money{} = new_count) do
old_count = old_count |> Money.to_decimal() |> Decimal.to_float() old_count = old_count |> Money.to_decimal() |> Decimal.to_float()

@ -208,12 +208,11 @@ defmodule Plausible.Stats.FilterSuggestions do
"operating_system" -> :operating_system "operating_system" -> :operating_system
"operating_system_version" -> :operating_system_version "operating_system_version" -> :operating_system_version
"screen_size" -> :screen_size "screen_size" -> :screen_size
"hostname" -> :hostname
_ -> :unknown _ -> :unknown
end end
q = q =
if(filter_name == :pathname or filter_name == :hostname, if(filter_name == :pathname,
do: base_event_query(site, query), do: base_event_query(site, query),
else: query_sessions(site, query) else: query_sessions(site, query)
) )
@ -231,12 +230,6 @@ defmodule Plausible.Stats.FilterSuggestions do
where: fragment("? ilike ?", e.pathname, ^filter_query) where: fragment("? ilike ?", e.pathname, ^filter_query)
) )
:hostname ->
from(e in q,
select: e.hostname,
where: fragment("? ilike ?", e.hostname, ^filter_query)
)
:entry_page -> :entry_page ->
from(e in q, from(e in q,
select: e.entry_page, select: e.entry_page,

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save