Compare commits

..

1 Commits

Author SHA1 Message Date
Karl-Aksel Puulmann da0638d198 Populate log_comment column with tags for performance analysis
These tags include information like plausible.site.id and http.route.

The tags will be exposed in system.query_log table log_comment column and can be used
for query performance analysis.

The tags are indirectly fetched from OpenTelemetry traces to avoid
duplicating data. The fetching mechanism is clunky and the best I
could figure out for now - help appreciated.
3 years ago

@ -1,73 +0,0 @@
# This file excludes paths from the Docker build context.
#
# By default, Docker's build context includes all files (and folders) in the
# current directory. Even if a file isn't copied into the container it is still sent to
# the Docker daemon.
#
# There are multiple reasons to exclude files from the build context:
#
# 1. Prevent nested folders from being copied into the container (ex: exclude
# /assets/node_modules when copying /assets)
# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc)
# 3. Avoid sending files containing sensitive information
#
# More information on using .dockerignore is available here:
# https://docs.docker.com/engine/reference/builder/#dockerignore-file
.dockerignore
# Ignore git, but keep git HEAD and refs to access current commit hash if needed:
#
# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat
# d0b8727759e1e0e7aa3d41707d12376e373d5ecc
.git
!.git/HEAD
!.git/refs
# Common development/test artifacts
/cover/
/doc/
/test/
/tmp/
.elixir_ls
plausible-report.xml
.env
.idea
*.iml
*.log
*.code-workspace
.vscode
# Mix artifacts
/_build/
/deps/
*.ez
# Generated on crash by the VM
erl_crash.dump
# If NPM crashes, it generates a log, let's ignore it too.
npm-debug.log
# Static artifacts - These should be fetched and built inside the Docker image
/assets/node_modules/
/tracker/node_modules/
/priv/static/cache_manifest.json
/priv/static/css
/priv/static/js
/priv/version.json
# Auto-generated tracker files
/priv/tracker/js/*.js
# Dializer
/priv/plts/*.plt
/priv/plts/*.plt.hash
# Geolocation databases
/priv/geodb/*.mmdb
/priv/geodb/*.mmdb.gz
# Docker volumes
.clickhouse_db_vol*
plausible_db*

@ -1,67 +0,0 @@
name: Build Private Images GHCR
on:
push:
branches: [master, stable]
tags: ['r*']
pull_request:
types: [synchronize, labeled]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
if: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'deploy-to-staging') }}
runs-on: buildjet-16vcpu-ubuntu-2204
permissions:
contents: read
packages: write
steps:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5.0.0
with:
images: ghcr.io/plausible/analytics/ee
tags: |
type=ref,event=pr
type=ref,event=branch
type=ref,event=tag
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
MIX_ENV=prod
BUILD_METADATA=${{ steps.meta.outputs.json }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
- name: Notify team on failure
if: ${{ failure() }}
uses: fjogeleit/http-request-action@v1
with:
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
method: 'POST'
customHeaders: '{"Content-Type": "application/json"}'
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-private-images.yml\">Build failed</a>"}'

@ -5,6 +5,7 @@ on:
branches: [master, stable]
tags: ['r*']
pull_request:
branches: [master]
types: [synchronize, labeled]
concurrency:

@ -1,68 +0,0 @@
name: Build Public Images GHCR
on:
push:
tags: ['v*']
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: buildjet-16vcpu-ubuntu-2204
permissions:
contents: read
packages: write
steps:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/plausible/community-edition
tags: |
type=semver,pattern={{version}},prefix=v
type=semver,pattern={{major}}.{{minor}},prefix=v
type=semver,pattern={{major}},prefix=v
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64, linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
MIX_ENV=small
BUILD_METADATA=${{ steps.meta.outputs.json }}
ERL_FLAGS=+JPperf true
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
- name: Notify team on failure
if: ${{ failure() }}
uses: fjogeleit/http-request-action@v1
with:
url: ${{ secrets.BUILD_NOTIFICATION_URL }}
method: 'POST'
customHeaders: '{"Content-Type": "application/json"}'
data: '{"content": "<a href=\"https://github.com/plausible/analytics/actions/workflows/build-public-images.yml\">Build failed</a>"}'

@ -17,7 +17,7 @@ jobs:
id: meta
uses: docker/metadata-action@v5
with:
images: plausible/community-edition
images: plausible/analytics
tags: |
type=semver,pattern={{version}},prefix=v
type=semver,pattern={{major}}.{{minor}},prefix=v
@ -48,7 +48,7 @@ jobs:
build-args: |
MIX_ENV=small
BUILD_METADATA=${{ steps.meta.outputs.json }}
ERL_FLAGS=+JMsingle true
ERL_FLAGS=+JPperf true
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}

@ -1,42 +1,27 @@
name: Elixir CI
on:
pull_request:
push:
branches: [master, stable]
pull_request:
branches: [master, stable]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CACHE_VERSION: v7
PERSISTENT_CACHE_DIR: cached
jobs:
build:
name: "Build and test (${{ matrix.mix_env }}, ${{ matrix.postgres_image }}${{ matrix.test_experimental_reduced_joins == '1' && ', experimental_reduced_joins' || '' }})"
name: Build and test
runs-on: ubuntu-latest
strategy:
matrix:
mix_env: ['test', 'small_test']
postgres_image: ['postgres:16']
test_experimental_reduced_joins: ['0']
postgres_version: [13, 14, 15]
include:
- mix_env: 'test'
postgres_image: 'postgres:15'
test_experimental_reduced_joins: '0'
- mix_env: 'test'
postgres_image: 'postgres:16'
test_experimental_reduced_joins: '1'
env:
MIX_ENV: ${{ matrix.mix_env }}
TEST_EXPERIMENTAL_REDUCED_JOINS: ${{ matrix.test_experimental_reduced_joins }}
services:
postgres:
image: ${{ matrix.postgres_image }}
image: postgres:${{ matrix.postgres_version }}
ports:
- 5432:5432
env:
@ -46,6 +31,7 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
clickhouse:
image: clickhouse/clickhouse-server:23.3.7.5-alpine
ports:
@ -56,91 +42,43 @@ jobs:
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: marocchino/tool-versions-action@v1
id: versions
- uses: erlef/setup-beam@v1
with:
elixir-version: ${{ steps.versions.outputs.elixir }}
otp-version: ${{ steps.versions.outputs.erlang }}
- uses: actions/cache@v4
with:
path: |
deps
_build
tracker/node_modules
priv/tracker/js
${{ env.PERSISTENT_CACHE_DIR }}
key: ${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
restore-keys: |
${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
- name: Check for changes in tracker/**
uses: dorny/paths-filter@v3
id: changes
with:
filters: |
tracker:
- 'tracker/**'
- name: Check if priv/tracker/js/plausible.js exists
run: |
if [ -f priv/tracker/js/plausible.js ]; then
echo "HAS_BUILT_TRACKER=true" >> $GITHUB_ENV
else
echo "HAS_BUILT_TRACKER=false" >> $GITHUB_ENV
fi
- run: npm install --prefix ./tracker
if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
- run: npm run deploy --prefix ./tracker
if: steps.changes.outputs.tracker == 'true' || env.HAS_BUILT_TRACKER == 'false'
- run: mix deps.get --only $MIX_ENV
- run: mix compile --warnings-as-errors --all-warnings
- run: mix do ecto.create, ecto.migrate
- run: mix run -e "Tzdata.ReleaseUpdater.poll_for_update"
- run: mix test --include slow --include minio --max-failures 1 --warnings-as-errors
if: env.MIX_ENV == 'test'
- run: mix test --include slow --max-failures 1 --warnings-as-errors
if: env.MIX_ENV == 'small_test'
static:
name: Static checks (format, credo, dialyzer)
env:
MIX_ENV: test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: marocchino/tool-versions-action@v1
- name: Read .tool-versions
uses: marocchino/tool-versions-action@v1
id: versions
- uses: erlef/setup-beam@v1
- name: Set up Elixir
uses: erlef/setup-beam@v1
with:
elixir-version: ${{ steps.versions.outputs.elixir }}
otp-version: ${{ steps.versions.outputs.erlang }}
- uses: actions/cache@v4
elixir-version: ${{steps.versions.outputs.elixir}}
otp-version: ${{ steps.versions.outputs.erlang}}
- name: Restore dependencies cache
uses: buildjet/cache@v3
with:
path: |
deps
_build
priv/plts
key: static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-${{ hashFiles('**/mix.lock') }}
restore-keys: |
static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-${{ github.head_ref || github.ref }}-
static-${{ env.MIX_ENV }}-${{ env.CACHE_VERSION }}-refs/heads/master-
- run: mix deps.get
- run: mix compile --warnings-as-errors --all-warnings
- run: mix format --check-formatted
- run: mix deps.unlock --check-unused
- run: mix credo diff --from-git-merge-base origin/master
- run: mix dialyzer
key: ${{ runner.os }}-mix-v7-${{ hashFiles('**/mix.lock') }}
restore-keys: ${{ runner.os }}-mix-v7-
- name: Install dependencies
run: mix deps.get && npm install --prefix ./tracker
- name: Check Formatting
run: mix format --check-formatted
- name: Check Compile Warnings
run: mix compile --warnings-as-errors --all-warnings
- name: Generate tracker files
run: npm run deploy --prefix ./tracker
- name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- name: Run tests
run: mix test --include slow --max-failures 1 --warnings-as-errors
- name: Run tests (small build)
run: MIX_ENV=small_test mix test --include slow --max-failures 1 --warnings-as-errors
- name: Check Dialyzer
run: mix dialyzer
env:
MIX_ENV: test

@ -1,28 +0,0 @@
name: Validate migrations
on:
pull_request:
paths:
- 'priv/repo/migrations/**'
- 'priv/ingest_repo/migrations/**'
jobs:
validate:
name: App code does not change at the same time
runs-on: ubuntu-latest
steps:
- uses: dorny/paths-filter@v3
id: changes
with:
list-files: json
filters: |
lib:
- 'lib/**'
- 'extra/**'
- 'config/**'
- if: steps.changes.outputs.lib == 'true'
run: |
echo "::error file=${{ fromJSON(steps.changes.outputs.lib_files)[0] }}::Code and migrations shouldn't be changed at the same time"
exit 1

@ -4,6 +4,7 @@ on:
push:
branches: [master, stable]
pull_request:
branches: [master, stable]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}

@ -35,7 +35,7 @@ jobs:
otp-version: ${{ steps.versions.outputs.erlang}}
- name: Restore Elixir dependencies cache
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: |
deps

@ -4,9 +4,11 @@ on:
push:
branches:
- master
paths:
- 'test/e2e/**.tf'
paths:
- 'test/e2e/**.tf'
pull_request:
branches:
- master
paths:
- 'test/e2e/**.tf'
@ -41,7 +43,7 @@ jobs:
- name: Terraform Init
id: init
run: terraform init
- name: Terraform Validate
id: validate
run: terraform validate -no-color

3
.gitignore vendored

@ -19,9 +19,6 @@ erl_crash.dump
# Also ignore archive artifacts (built via "mix archive.build").
*.ez
# Temporary files, for example, from tests.
/tmp/
# Ignore package tarball (built via "mix hex.build").
plausible-*.tar

@ -2,24 +2,6 @@
All notable changes to this project will be documented in this file.
### Added
- Pages Block List in Site Settings
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `total_conversions` and `conversion_rate` to `visitors.csv` in a goal-filtered CSV export
- Ability to display total conversions (with a goal filter) on the main graph
- Add `conversion_rate` to Stats API Timeseries and on the main graph
- Add `time_on_page` metric into the Stats API
- County Block List in Site Settings
- Query the `views_per_visit` metric based on imported data as well if possible
- Group `operating_system_versions` by `operating_system` in Stats API breakdown
- Add `operating_system_versions.csv` into the CSV export
- Display `Total visitors`, `Conversions`, and `CR` in the "Details" views of Countries, Regions and Cities (when filtering by a goal)
- Add `conversion_rate` to Regions and Cities reports (when filtering by a goal)
- Add the `conversion_rate` metric to Stats API Breakdown and Aggregate endpoints
- IP Block List in Site Settings
- Allow filtering with `contains`/`matches` operator for Sources, Browsers and Operating Systems.
- Allow filtering by multiple custom properties
- Wildcard and member filtering on the Stats API `event:goal` property
- Allow filtering with `contains`/`matches` operator for custom properties
- Add `referrers.csv` to CSV export
- Add a new Properties section in the dashboard to break down by custom properties
- Add `custom_props.csv` to CSV export (almost the same as the old `prop_breakdown.csv`, but has different column headers, and includes props for pageviews too, not only custom events)
@ -35,34 +17,20 @@ All notable changes to this project will be documented in this file.
- Add support for 2FA authentication
- Add 'browser_versions.csv' to CSV export
- Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB)
- Add alternative SMTP adapter plausible/analytics#3654
- Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906
- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
### Removed
- Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions
- Removed the `prop_names` returned in the Stats API `event:goal` breakdown response
- Removed the `prop-breakdown.csv` file from CSV export
- Deprecated `CLICKHOUSE_MAX_BUFFER_SIZE`
- Removed `/app/init-admin.sh` that was deprecated in v2.0.0 plausible/analytics#3903
- Remove `DISABLE_AUTH` deprecation warning plausible/analytics#3904
### Changed
- A visits `entry_page` and `exit_page` is only set and updated for pageviews, not custom events
- Limit the number of Goal Conversions shown on the dashboard and render a "Details" link when there are more entries to show
- Show Outbound Links / File Downloads / 404 Pages / Cloaked Links instead of Goal Conversions when filtering by the corresponding goal
- Require custom properties to be explicitly added from Site Settings > Custom Properties in order for them to show up on the dashboard
- GA/SC sections moved to new settings: Integrations
- Replace `CLICKHOUSE_MAX_BUFFER_SIZE` with `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES`
- Validate metric isn't queried multiple times
### Fixed
- Creating many sites no longer leads to cookie overflow
- Ignore sessions without pageviews for `entry_page` and `exit_page` breakdowns
- Using `VersionedCollapsingMergeTree` to store visit data to avoid rare race conditions that led to wrong visit data being shown
- Fix `conversion_rate` metric in a `browser_versions` breakdown
- Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division)
- Calculate `bounce_rate` percentage change in the Stats API in the same way as it's done in the dashboard
- Stop returning custom events in goal breakdown with a pageview goal filter and vice versa
- Only return `(none)` values in custom property breakdown for the first page (pagination) of results
- Fixed weekly/monthly e-mail report [rendering issues](https://github.com/plausible/analytics/issues/284)
@ -76,11 +44,6 @@ All notable changes to this project will be documented in this file.
- Log mailer errors plausible/analytics#3336
- Allow custom event timeseries in stats API plausible/analytics#3505
- Fixes for sites with UTF characters in domain plausible/analytics#3560
- Fix crash when using special characters in filter plausible/analytics#3634
- Fix automatic scrolling to the bottom on the dashboard if previously selected properties tab plausible/analytics#3872
- Allow running the container with arbitrary UID plausible/analytics#2986
- Fix `width=manual` in embedded dashboards plausible/analytics#3910
- Fix URL escaping when pipes are used in UTM tags plausible/analytics#3930
## v2.0.0 - 2023-07-12

@ -47,7 +47,8 @@ RUN npm run deploy --prefix ./tracker && \
mix assets.deploy && \
mix phx.digest priv/static && \
mix download_country_database && \
mix sentry.package_source_code
# https://hexdocs.pm/sentry/Sentry.Sources.html#module-source-code-storage
mix sentry_recompile
WORKDIR /app
COPY rel rel
@ -63,15 +64,18 @@ ENV LANG=C.UTF-8
ARG MIX_ENV=small
ENV MIX_ENV=$MIX_ENV
RUN adduser -S -H -u 999 -G nogroup plausible -g 'Plausible Analytics'
RUN apk upgrade --no-cache
RUN apk add --no-cache openssl ncurses libstdc++ libgcc ca-certificates
COPY --from=buildcontainer --chmod=a+rX /app/_build/${MIX_ENV}/rel/plausible /app
COPY --chmod=755 ./rel/docker-entrypoint.sh /entrypoint.sh
COPY ./rel/docker-entrypoint.sh /entrypoint.sh
RUN chmod a+x /entrypoint.sh && \
adduser -h /app -u 1000 -s /bin/sh -D plausibleuser
USER 999
COPY --from=buildcontainer /app/_build/${MIX_ENV}/rel/plausible /app
RUN chown -R plausibleuser:plausibleuser /app
USER plausibleuser
WORKDIR /app
ENV LISTEN_IP=0.0.0.0
ENTRYPOINT ["/entrypoint.sh"]

@ -18,7 +18,7 @@ server: ## Start the web server
CH_FLAGS ?= --detach -p 8123:8123 -p 9000:9000 --ulimit nofile=262144:262144 --name plausible_clickhouse
clickhouse: ## Start a container with a recent version of clickhouse
docker run $(CH_FLAGS) --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:latest-alpine
docker run $(CH_FLAGS) --volume=$$PWD/.clickhouse_db_vol:/var/lib/clickhouse clickhouse/clickhouse-server:22.9-alpine
clickhouse-prod: ## Start a container with the same version of clickhouse as the one in prod
docker run $(CH_FLAGS) --volume=$$PWD/.clickhouse_db_vol_prod:/var/lib/clickhouse clickhouse/clickhouse-server:23.3.7.5-alpine
@ -36,12 +36,3 @@ postgres-prod: ## Start a container with the same version of postgres as the one
postgres-stop: ## Stop and remove the postgres container
docker stop plausible_db && docker rm plausible_db
minio: ## Start a transient container with a recent version of minio (s3)
docker run -d --rm -p 10000:10000 -p 10001:10001 --name plausible_minio minio/minio server /data --address ":10000" --console-address ":10001"
while ! docker exec plausible_minio mc alias set local http://localhost:10000 minioadmin minioadmin; do sleep 1; done
docker exec plausible_minio mc mb local/dev-exports
docker exec plausible_minio mc mb local/dev-imports
minio-stop: ## Stop and remove the minio container
docker stop plausible_minio

@ -15,7 +15,7 @@
<br /><br />
</p>
[Plausible Analytics](https://plausible.io/) is an easy to use, lightweight (< 1 KB), open source and privacy-friendly alternative to Google Analytics. It doesn’t use cookies and is fully compliant with GDPR, CCPA and PECR. You can self-host Plausible Community Edition or have us manage Plausible Analytics for you in the cloud. Here's [the live demo of our own website stats](https://plausible.io/plausible.io). Made and hosted in the EU 🇪🇺
[Plausible Analytics](https://plausible.io/) is an easy to use, lightweight (< 1 KB), open source and privacy-friendly alternative to Google Analytics. It doesn’t use cookies and is fully compliant with GDPR, CCPA and PECR. You can self-host or have us run Plausible for you in the cloud. Here's [the live demo of our own website stats](https://plausible.io/plausible.io). Made and hosted in the EU 🇪🇺
We are dedicated to making web analytics more privacy-friendly. Our mission is to reduce corporate surveillance by providing an alternative web analytics tool which doesn’t come from the AdTech world. We are completely independent and solely funded by our subscribers.
@ -23,7 +23,7 @@ We are dedicated to making web analytics more privacy-friendly. Our mission is t
## Why Plausible?
Here's what makes Plausible a great Google Analytics alternative and why we're trusted by 12,000+ paying subscribers to deliver their website and business insights:
Here's what makes Plausible a great Google Analytics alternative and why we're trusted by 11,000+ paying subscribers to deliver their website and business insights:
- **Clutter Free**: Plausible Analytics provides [simple web analytics](https://plausible.io/simple-web-analytics) and it cuts through the noise. No layers of menus, no need for custom reports. Get all the important insights on one single page. No training necessary.
- **GDPR/CCPA/PECR compliant**: Measure traffic, not individuals. No personal data or IP addresses are ever stored in our database. We don't use cookies or any other persistent identifiers. [Read more about our data policy](https://plausible.io/data-policy)
@ -49,32 +49,32 @@ We choose the subscription business model rather than the business model of surv
## Getting started with Plausible
The easiest way to get started with Plausible Analytics is with [our official managed service in the cloud](https://plausible.io/#pricing). It takes 2 minutes to start counting your stats with a worldwide CDN, high availability, backups, security and maintenance all done for you by us.
The easiest way to get started with Plausible is with [our official managed service in the cloud](https://plausible.io/#pricing). It takes 2 minutes to start counting your stats with a worldwide CDN, high availability, backups, security and maintenance all done for you by us.
In order to be compliant with the GDPR and the Schrems II ruling, all visitor data for our managed service in the cloud is exclusively processed on servers and cloud infrastructure owned and operated by European providers. Your website data never leaves the EU.
Our managed hosting can save a substantial amount of developer time and resources. For most sites this ends up being the best value option and the revenue goes to funding the maintenance and further development of Plausible. So you’ll be supporting open source software and getting a great service!
### Can Plausible be self-hosted?
### Can Plausible Analytics be self-hosted?
Plausible is [open source web analytics](https://plausible.io/open-source-website-analytics) and we have a free as in beer and self-hosted solution called [Plausible Community Edition (CE)](https://plausible.io/self-hosted-web-analytics). Here are the differences between Plausible Analytics managed hosting in the cloud and the Plausible CE:
Plausible is [open source web analytics](https://plausible.io/open-source-website-analytics) and we have a free as in beer [self-hosted solution](https://plausible.io/self-hosted-web-analytics). Here are the differences between Plausible Analytics managed hosting in the cloud and the self-hosting:
| | Plausible Analytics Cloud | Plausible Community Edition |
| | Plausible Cloud | Self-Hosting |
| ------------- | ------------- | ------------- |
| **Infrastructure management** | Easy and convenient. It takes 2 minutes to start counting your stats with a worldwide CDN, high availability, backups, security and maintenance all done for you by us. We manage everything so you don’t have to worry about anything and can focus on your stats. | You do it all yourself. You need to get a server and you need to manage your infrastructure. You are responsible for installation, maintenance, upgrades, server capacity, uptime, backup, security, stability, consistency, loading time and so on.|
| **Release schedule** | Continuously developed and improved with new features and updates multiple times per week. | [It's a long term release](https://plausible.io/blog/building-open-source) published twice per year so latest features and improvements won't be immediately available.|
| **Premium features** | All features available as listed in [our pricing plans](https://plausible.io/#pricing). | Selected premium features such as funnels and ecommerce revenue goals are not available as we aim to ensure a [protective barrier around our cloud offering](https://plausible.io/blog/community-edition).|
| **Premium features** | All features available as listed in [our pricing plans](https://plausible.io/#pricing). | Selected premium features such as funnels and ecommerce revenue goals are not available as we aim to ensure a protective barrier around our cloud offering.|
| **Bot filtering** | Advanced bot filtering for more accurate stats. Our algorithm detects and excludes non-human traffic patterns. We also exclude known bots by the User-Agent header and filter out traffic from data centers and referrer spam domains. | Basic bot filtering that targets the most common non-human traffic based on the User-Agent header and referrer spam domains.|
| **Server location** | All visitor data is exclusively processed on EU-owned cloud infrastructure. We keep your site data on a secure, encrypted and green energy powered server in Germany. This ensures that your site data is protected by the strict European Union data privacy laws and ensures compliance with GDPR. Your website data never leaves the EU. | You have full control and can host your instance on any server in any country that you wish. Host it on a server in your basement or host it with any cloud provider wherever you want, even those that are not GDPR compliant.|
| **Data portability** | You see all your site stats and metrics on our modern-looking, simple to use and fast loading dashboard. You can only see the stats aggregated in the dashboard. You can download the stats using the [CSV export](https://plausible.io/docs/export-stats), [stats API](https://plausible.io/docs/stats-api) or tools such as the [Data Studio Connector](https://plausible.io/docs/integration-guides#google-data-studio). | Do you want access to the raw data? Self-hosting gives you that option. You can take the data directly from the ClickHouse database. |
| **Premium support** | Real support delivered by real human beings who build and maintain Plausible. | Premium support is not included. CE is community supported only.|
| **Premium support** | Real support delivered by real human beings who build and maintain Plausible. | Premium support is not included. Self-hosting is community supported only.|
| **Costs** | There's a cost associated with providing an analytics service so we charge a subscription fee. We choose the subscription business model rather than the business model of surveillance capitalism. Your money funds further development of Plausible. | You need to pay for your server, CDN, backups and whatever other cost there is associated with running the infrastructure. You never have to pay any fees to us. Your money goes to 3rd party companies with no connection to us.|
Interested in self-hosting Plausible CE on your server? Take a look at our [Plausible CE installation instructions](https://github.com/plausible/community-edition/).
Interested in self-hosting Plausible on your server? Take a look at our [self-hosting installation instructions](https://plausible.io/docs/self-hosting).
Plausible CE is a community supported project and there are no guarantees that you will get support from the creators of Plausible to troubleshoot your self-hosting issues. There is a [community supported forum](https://github.com/plausible/analytics/discussions/categories/self-hosted-support) where you can ask for help.
Plausible Self-Hosted is a community supported project and there are no guarantees that you will get support from the creators of Plausible to troubleshoot your self-hosting issues. There is a [community supported forum](https://github.com/plausible/analytics/discussions/categories/self-hosted-support) where you can ask for help.
Our only source of funding is our premium, managed service for running Plausible in the cloud.
Our only source of funding is our premium, managed service for running Plausible in the cloud. If you're looking for an alternative way to support the project, we've put together some sponsorship packages. If you choose to self-host Plausible you can [become a sponsor](https://github.com/sponsors/plausible) which is a great way to give back to the community and to contribute to the long-term sustainability of the project.
## Technology
@ -95,7 +95,7 @@ To stay up to date with all the latest news and product updates, make sure to fo
## License & Trademarks
Plausible CE is open source under the GNU Affero General Public License Version 3 (AGPLv3) or any later version. You can [find it here](https://github.com/plausible/analytics/blob/master/LICENSE.md).
Plausible is open source under the GNU Affero General Public License Version 3 (AGPLv3) or any later version. You can [find it here](https://github.com/plausible/analytics/blob/master/LICENSE.md).
To avoid issues with AGPL virality, we've released the JavaScript tracker which gets included on your website under the MIT license. You can [find it here](https://github.com/plausible/analytics/blob/master/tracker/LICENSE.md).

@ -9,13 +9,6 @@
[x-cloak] { display: none; }
@media print {
canvas {
width: 100% !important;
height: auto !important;
}
}
.button {
@apply inline-flex justify-center px-4 py-2 text-sm font-medium text-white bg-indigo-600 border border-transparent rounded-md leading-5 transition hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500;
}

@ -10,16 +10,6 @@ Alpine.data('dropdown', dropdown)
Alpine.data('comboBox', comboBox)
Alpine.start()
if (document.querySelectorAll('[data-modal]').length > 0) {
window.addEventListener(`phx:close-modal`, (e) => {
document
.getElementById(e.detail.id)
.dispatchEvent(
new CustomEvent('close-modal', { bubbles: true, detail: e.detail.id })
)
})
}
const triggers = document.querySelectorAll('[data-dropdown-trigger]')
for (const trigger of triggers) {

@ -22,7 +22,6 @@ if (container) {
conversionsOptedOut: container.dataset.conversionsOptedOut === 'true',
funnelsOptedOut: container.dataset.funnelsOptedOut === 'true',
propsOptedOut: container.dataset.propsOptedOut === 'true',
revenueGoals: JSON.parse(container.dataset.revenueGoals),
funnels: JSON.parse(container.dataset.funnels),
statsBegin: container.dataset.statsBegin,
nativeStatsBegin: container.dataset.nativeStatsBegin,

@ -1,4 +1,4 @@
import { formatISO } from './util/date'
import {formatISO} from './util/date'
let abortController = new AbortController()
let SHARED_LINK_AUTH = null
@ -36,15 +36,14 @@ function serializeFilters(filters) {
return JSON.stringify(cleaned)
}
export function serializeQuery(query, extraQuery = []) {
export function serializeQuery(query, extraQuery=[]) {
const queryObj = {}
if (query.period) { queryObj.period = query.period }
if (query.date) { queryObj.date = formatISO(query.date) }
if (query.from) { queryObj.from = formatISO(query.from) }
if (query.to) { queryObj.to = formatISO(query.to) }
if (query.filters) { queryObj.filters = serializeFilters(query.filters) }
if (query.experimental_session_count) { queryObj.experimental_session_count = query.experimental_session_count }
if (query.with_imported) { queryObj.with_imported = query.with_imported }
if (query.period) { queryObj.period = query.period }
if (query.date) { queryObj.date = formatISO(query.date) }
if (query.from) { queryObj.from = formatISO(query.from) }
if (query.to) { queryObj.to = formatISO(query.to) }
if (query.filters) { queryObj.filters = serializeFilters(query.filters) }
if (query.with_imported) { queryObj.with_imported = query.with_imported }
if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH }
if (query.comparison) {
@ -59,11 +58,11 @@ export function serializeQuery(query, extraQuery = []) {
return '?' + serialize(queryObj)
}
export function get(url, query = {}, ...extraQuery) {
const headers = SHARED_LINK_AUTH ? { 'X-Shared-Link-Auth': SHARED_LINK_AUTH } : {}
export function get(url, query={}, ...extraQuery) {
const headers = SHARED_LINK_AUTH ? {'X-Shared-Link-Auth': SHARED_LINK_AUTH} : {}
url = url + serializeQuery(query, extraQuery)
return fetch(url, { signal: abortController.signal, headers: headers })
.then(response => {
return fetch(url, {signal: abortController.signal, headers: headers})
.then( response => {
if (!response.ok) {
return response.json().then((msg) => {
throw new ApiError(msg.error, msg)

@ -91,10 +91,8 @@ export default function PlausibleCombobox(props) {
}
}
function isOptionDisabled(option) {
const optionAlreadySelected = props.values.some((val) => val.value === option.value)
const optionDisabled = (props.disabledOptions || []).some((val) => val?.value === option.value)
return optionAlreadySelected || optionDisabled
function isDisabled(option) {
return props.values.some((val) => val.value === option.value)
}
function fetchOptions(query) {
@ -159,11 +157,11 @@ export default function PlausibleCombobox(props) {
}, [])
useEffect(() => {
if (props.singleOption && props.values.length === 0 && props.autoFocus) {
if (props.singleOption && props.values.length === 0) {
searchRef.current.focus()
}
}, [props.values.length === 0, props.singleOption, props.autoFocus])
}, [props.values.length === 0])
const searchBoxClass = 'border-none py-1 px-0 w-full inline-block rounded-md focus:outline-none focus:ring-0 text-sm'
const containerClass = classNames('relative w-full', {
@ -219,15 +217,15 @@ export default function PlausibleCombobox(props) {
}
function renderDropDownContent() {
const matchesFound = visibleOptions.length > 0 && visibleOptions.some(option => !isOptionDisabled(option))
const matchesFound = visibleOptions.length > 0 && visibleOptions.some(option => !isDisabled(option))
if (loading) {
return <div className="relative cursor-default select-none py-2 px-4 text-gray-700 dark:text-gray-300">Loading options...</div>
}
if (matchesFound) {
return visibleOptions
.filter(option => !isOptionDisabled(option))
.filter(option => !isDisabled(option))
.map((option, i) => {
const text = option.freeChoice ? `Filter by '${option.label}'` : option.label

@ -1,6 +1,6 @@
import React, { Fragment } from "react";
import { FILTER_OPERATIONS } from "../util/filters";
import { FILTER_TYPES } from "../util/filters";
import { Menu, Transition } from "@headlessui/react";
import { ChevronDownIcon } from '@heroicons/react/20/solid'
import { isFreeChoiceFilter, supportsIsNot } from "../util/filters";
@ -61,9 +61,9 @@ export default function FilterTypeSelector(props) {
className="z-10 origin-top-left absolute left-0 mt-2 w-full rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none"
>
<div className="py-1">
{renderTypeItem(FILTER_OPERATIONS.is, true)}
{renderTypeItem(FILTER_OPERATIONS.isNot, supportsIsNot(filterName))}
{renderTypeItem(FILTER_OPERATIONS.contains, isFreeChoiceFilter(filterName))}
{renderTypeItem(FILTER_TYPES.is, true)}
{renderTypeItem(FILTER_TYPES.isNot, supportsIsNot(filterName))}
{renderTypeItem(FILTER_TYPES.contains, isFreeChoiceFilter(filterName))}
</div>
</Menu.Items>
</Transition>

@ -1,5 +0,0 @@
Plausible copyright notice
This software is Copyright 2024 Plausible Insights OÜ
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io

@ -10,23 +10,12 @@ import {
formatFilterGroup,
filterGroupForFilter,
parseQueryFilter,
parseQueryPropsFilter,
formattedFilters
} from "./util/filters";
function removeFilter(filterType, key, history, query) {
const newOpts = {}
if (filterType === 'props') {
if (Object.keys(query.filters.props).length == 1) {
newOpts.props = false
} else {
newOpts.props = JSON.stringify({
...query.filters.props,
[key]: undefined,
})
}
} else {
newOpts[key] = false
function removeFilter(key, history, query) {
const newOpts = {
[key]: false
}
if (key === 'country') { newOpts.country_labels = false }
if (key === 'region') { newOpts.region_labels = false }
@ -48,38 +37,34 @@ function clearAllFilters(history, query) {
);
}
function filterText(filterType, key, query) {
function filterText(key, _rawValue, query) {
const {type, clauses} = parseQueryFilter(query, key)
const formattedFilter = formattedFilters[key]
if (filterType === "props") {
const { propKey, clauses, type } = parseQueryPropsFilter(query).find((filter) => filter.propKey.value === key)
return <>Property <b>{propKey.label}</b> {type} {clauses.map(({label}) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
if (key === "props") {
const [[propKey, _propValue]] = Object.entries(query.filters['props'])
return <>Property <b>{propKey}</b> {type} {clauses.map(({label}) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
} else if (formattedFilter) {
const {type, clauses} = parseQueryFilter(query, key)
return <>{formattedFilter} {type} {clauses.map(({label}) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
}
throw new Error(`Unknown filter: ${key}`)
}
function renderDropdownFilter(site, history, { key, value, filterType }, query) {
function renderDropdownFilter(site, history, [key, value], query) {
return (
<Menu.Item key={`${filterType}::${key}`}>
<Menu.Item key={key}>
<div className="px-3 md:px-4 sm:py-2 py-3 text-sm leading-tight flex items-center justify-between" key={key + value}>
<Link
title={`Edit filter: ${formattedFilters[filterType]}`}
to={{ pathname: `/${encodeURIComponent(site.domain)}/filter/${filterGroupForFilter(filterType)}`, search: window.location.search }}
title={`Edit filter: ${formattedFilters[key]}`}
to={{ pathname: `/${encodeURIComponent(site.domain)}/filter/${filterGroupForFilter(key)}`, search: window.location.search }}
className="group flex w-full justify-between items-center"
style={{ width: 'calc(100% - 1.5rem)' }}
>
<span className="inline-block w-full truncate">{filterText(filterType, key, query)}</span>
<span className="inline-block w-full truncate">{filterText(key, value, query)}</span>
<PencilSquareIcon className="w-4 h-4 ml-1 cursor-pointer group-hover:text-indigo-700 dark:group-hover:text-indigo-500" />
</Link>
<b
title={`Remove filter: ${formattedFilters[filterType]}`}
className="ml-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500"
onClick={() => removeFilter(filterType, key, history, query)}
>
<b title={`Remove filter: ${formattedFilters[key]}`} className="ml-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500" onClick={() => removeFilter(key, history, query)}>
<XMarkIcon className="w-4 h-4" />
</b>
</div>
@ -217,21 +202,13 @@ class Filters extends React.Component {
});
};
renderListFilter(history, { key, value, filterType }, query) {
renderListFilter(history, [key, value], query) {
return (
<span key={`${filterType}::${key}`} title={value} className="flex bg-white dark:bg-gray-800 text-gray-700 dark:text-gray-300 shadow text-sm rounded mr-2 items-center">
<Link
title={`Edit filter: ${formattedFilters[filterType]}`}
className="flex w-full h-full items-center py-2 pl-3"
to={{ pathname: `/${encodeURIComponent(this.props.site.domain)}/filter/${filterGroupForFilter(filterType)}`, search: window.location.search }}
>
<span className="inline-block max-w-2xs md:max-w-xs truncate">{filterText(filterType, key, query)}</span>
<span key={key} title={value} className="flex bg-white dark:bg-gray-800 text-gray-700 dark:text-gray-300 shadow text-sm rounded mr-2 items-center">
<Link title={`Edit filter: ${formattedFilters[key]}`} className="flex w-full h-full items-center py-2 pl-3" to={{ pathname: `/${encodeURIComponent(this.props.site.domain)}/filter/${filterGroupForFilter(key)}`, search: window.location.search }}>
<span className="inline-block max-w-2xs md:max-w-xs truncate">{filterText(key, value, query)}</span>
</Link>
<span
title={`Remove filter: ${formattedFilters[filterType]}`}
className="flex h-full w-full px-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500 items-center"
onClick={() => removeFilter(filterType, key, history, query)}
>
<span title={`Remove filter: ${formattedFilters[key]}`} className="flex h-full w-full px-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500 items-center" onClick={() => removeFilter(key, history, query)}>
<XMarkIcon className="w-4 h-4" />
</span>
</span>
@ -258,10 +235,6 @@ class Filters extends React.Component {
)
}
trackFilterMenu() {
window.plausible && window.plausible('Filter Menu: Open', {u: `${window.location.protocol}//${window.location.hostname}/:dashboard`})
}
renderDropDown() {
const { history, query, site } = this.props;
@ -270,7 +243,7 @@ class Filters extends React.Component {
{({ open }) => (
<>
<div>
<Menu.Button onClick={this.trackFilterMenu} className="flex items-center text-xs md:text-sm font-medium leading-tight px-3 py-2 cursor-pointer ml-auto text-gray-500 dark:text-gray-200 hover:bg-gray-200 dark:hover:bg-gray-900 rounded">
<Menu.Button className="flex items-center text-xs md:text-sm font-medium leading-tight px-3 py-2 cursor-pointer ml-auto text-gray-500 dark:text-gray-200 hover:bg-gray-200 dark:hover:bg-gray-900 rounded">
{this.renderDropdownButton()}
</Menu.Button>
</div>

@ -1,8 +1,8 @@
import React from 'react'
import { Link, withRouter } from 'react-router-dom'
import { nowForSite } from './util/date'
import {nowForSite} from './util/date'
import * as storage from './util/storage'
import { COMPARISON_DISABLED_PERIODS, getStoredComparisonMode, isComparisonEnabled, getStoredMatchDayOfWeek } from './comparison-input'
import { COMPARISON_DISABLED_PERIODS, getStoredComparisonMode, isComparisonEnabled,getStoredMatchDayOfWeek } from './comparison-input'
import dayjs from 'dayjs';
import utc from 'dayjs/plugin/utc';
@ -39,7 +39,6 @@ export function parseQuery(querystring, site) {
to: q.get('to') ? dayjs.utc(q.get('to')) : undefined,
match_day_of_week: matchDayOfWeek == 'true',
with_imported: q.get('with_imported') ? q.get('with_imported') === 'true' : true,
experimental_session_count: q.get('experimental_session_count'),
filters: {
'goal': q.get('goal'),
'props': JSON.parse(q.get('props')),
@ -59,7 +58,6 @@ export function parseQuery(querystring, site) {
'region': q.get('region'),
'city': q.get('city'),
'page': q.get('page'),
'hostname': q.get('hostname'),
'entry_page': q.get('entry_page'),
'exit_page': q.get('exit_page')
}
@ -67,13 +65,9 @@ export function parseQuery(querystring, site) {
}
export function appliedFilters(query) {
const propKeys = Object.entries(query.filters.props || {})
.map(([key, value]) => ({ key, value, filterType: 'props' }))
return Object.entries(query.filters)
.map(([key, value]) => ({ key, value, filterType: key }))
.filter(({ key, value }) => key !== 'props' && !!value)
.concat(propKeys)
return Object.keys(query.filters)
.map((key) => [key, query.filters[key]])
.filter(([_key, value]) => !!value);
}
function generateQueryString(data) {
@ -128,7 +122,7 @@ class QueryLink extends React.Component {
const QueryLinkWithRouter = withRouter(QueryLink)
export { QueryLinkWithRouter as QueryLink };
function QueryButton({ history, query, to, disabled, className, children, onClick }) {
function QueryButton({history, query, to, disabled, className, children, onClick}) {
return (
<button
className={className}

@ -52,7 +52,7 @@ export default function Router({ site, loggedIn, currentUserRole }) {
<ExitPagesModal site={site} />
</Route>
<Route path="/:domain/countries">
<ModalTable title="Top countries" site={site} endpoint={url.apiPath(site, '/countries')} filter={{ country: 'code', country_labels: 'name' }} keyLabel="Country" renderIcon={renderCountryIcon} showPercentage={true}/>
<ModalTable title="Top countries" site={site} endpoint={url.apiPath(site, '/countries')} filter={{ country: 'code', country_labels: 'name' }} keyLabel="Country" renderIcon={renderCountryIcon} />
</Route>
<Route path="/:domain/regions">
<ModalTable title="Top regions" site={site} endpoint={url.apiPath(site, '/regions')} filter={{ region: 'code', region_labels: 'name' }} keyLabel="Region" renderIcon={renderRegionIcon} />

@ -1,4 +1,4 @@
import React, { useCallback, useState } from "react"
import React, { useCallback, useState, useEffect } from "react";
import ListReport from "../reports/list";
import Combobox from '../../components/combobox'
import * as api from '../../api'
@ -15,6 +15,10 @@ export default function Properties(props) {
const [propKey, setPropKey] = useState(choosePropKey())
useEffect(() => {
setPropKey(choosePropKey())
}, [query.filters.goal, query.filters.props])
function singleGoalFilterApplied() {
const goalFilter = query.filters.goal
if (goalFilter) {
@ -88,16 +92,14 @@ export default function Properties(props) {
)
}
const getFilterFor = (listItem) => ({
props: JSON.stringify({ ...query.filters.props, [propKey]: escapeFilterValue(listItem.name) })
})
const getFilterFor = (listItem) => { return { 'props': JSON.stringify({ [propKey]: escapeFilterValue(listItem.name) }) } }
const comboboxValues = propKey ? [{ value: propKey, label: propKey }] : []
const boxClass = 'pl-2 pr-8 py-1 bg-transparent dark:text-gray-300 rounded-md shadow-sm border border-gray-300 dark:border-gray-500'
return (
<div className="w-full mt-4">
<div>
<Combobox boxClass={boxClass} fetchOptions={fetchPropKeyOptions()} singleOption={true} values={comboboxValues} onSelect={onPropKeySelect()} placeholder={'Select a property'} />
<Combobox isDisabled={!!query.filters.props} boxClass={boxClass} fetchOptions={fetchPropKeyOptions()} singleOption={true} values={comboboxValues} onSelect={onPropKeySelect()} placeholder={'Select a property'} />
</div>
{propKey && renderBreakdown()}
</div>

@ -1,7 +1,6 @@
import React from 'react';
import * as storage from '../../util/storage'
import { isFilteringOnFixedValue } from '../../util/filters'
import ListReport from '../reports/list'
import * as api from '../../api'
import * as url from '../../util/url'
@ -162,12 +161,12 @@ export default class Devices extends React.Component {
renderContent() {
switch (this.state.mode) {
case 'browser':
if (isFilteringOnFixedValue(this.props.query, 'browser')) {
if (this.props.query.filters.browser) {
return <BrowserVersions site={this.props.site} query={this.props.query} />
}
return <Browsers site={this.props.site} query={this.props.query} />
case 'os':
if (isFilteringOnFixedValue(this.props.query, 'os')) {
if (this.props.query.filters.os) {
return <OperatingSystemVersions site={this.props.site} query={this.props.query} />
}
return <OperatingSystems site={this.props.site} query={this.props.query} />

@ -1,53 +1,27 @@
import numberFormatter, {durationFormatter} from '../../util/number-formatter'
import { parsePrefix } from '../../util/filters'
export function getGraphableMetrics(query, site) {
const isRealtime = query.period === 'realtime'
const goalFilter = query.filters.goal
const pageFilter = query.filters.page
if (isRealtime && goalFilter) {
return ["visitors"]
} else if (isRealtime) {
return ["visitors", "pageviews"]
} else if (goalFilter && canGraphRevenueMetrics(goalFilter, site)) {
return ["visitors", "events", "average_revenue", "total_revenue", "conversion_rate"]
} else if (goalFilter) {
return ["visitors", "events", "conversion_rate"]
} else if (pageFilter) {
return ["visitors", "visits", "pageviews", "bounce_rate", "time_on_page"]
} else {
return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]
}
}
// Revenue metrics can only be graphed if:
// * The query is filtered by at least one revenue goal
// * All revenue goals in filter have the same currency
function canGraphRevenueMetrics(goalFilter, site) {
const goalsInFilter = parsePrefix(goalFilter).values
const revenueGoalsInFilter = site.revenueGoals.filter((rg) => {
return goalsInFilter.includes(rg.event_name)
})
const singleCurrency = revenueGoalsInFilter.every((rg) => {
return rg.currency === revenueGoalsInFilter[0].currency
})
return revenueGoalsInFilter.length > 0 && singleCurrency
export const METRIC_MAPPING = {
'Unique visitors (last 30 min)': 'visitors',
'Pageviews (last 30 min)': 'pageviews',
'Unique visitors': 'visitors',
'Visit duration': 'visit_duration',
'Total pageviews': 'pageviews',
'Views per visit': 'views_per_visit',
'Total visits': 'visits',
'Bounce rate': 'bounce_rate',
'Unique conversions': 'conversions',
'Average revenue': 'average_revenue',
'Total revenue': 'total_revenue',
}
export const METRIC_LABELS = {
'visitors': 'Visitors',
'pageviews': 'Pageviews',
'events': 'Total Conversions',
'views_per_visit': 'Views per Visit',
'visits': 'Visits',
'bounce_rate': 'Bounce Rate',
'visit_duration': 'Visit Duration',
'conversions': 'Converted Visitors',
'conversion_rate': 'Conversion Rate',
'average_revenue': 'Average Revenue',
'total_revenue': 'Total Revenue',
}
@ -55,17 +29,23 @@ export const METRIC_LABELS = {
export const METRIC_FORMATTER = {
'visitors': numberFormatter,
'pageviews': numberFormatter,
'events': numberFormatter,
'visits': numberFormatter,
'views_per_visit': (number) => (number),
'bounce_rate': (number) => (`${number}%`),
'visit_duration': durationFormatter,
'conversions': numberFormatter,
'conversion_rate': (number) => (`${number}%`),
'total_revenue': numberFormatter,
'average_revenue': numberFormatter,
}
export const LoadingState = {
loading: 'loading',
refreshing: 'refreshing',
loaded: 'loaded',
isLoadingOrRefreshing: function (state) { return [this.loading, this.refreshing].includes(state) },
isLoadedOrRefreshing: function (state) { return [this.loaded, this.refreshing].includes(state) }
}
const buildComparisonDataset = function(comparisonPlot) {
if (!comparisonPlot) return []

@ -6,7 +6,7 @@ import * as storage from '../../util/storage'
import { isKeyPressed } from '../../keybinding.js'
import { monthsBetweenDates } from '../../util/date.js'
const INTERVAL_LABELS = {
export const INTERVAL_LABELS = {
'minute': 'Minutes',
'hour': 'Hours',
'date': 'Days',
@ -14,19 +14,11 @@ const INTERVAL_LABELS = {
'month': 'Months'
}
function validIntervals(site, query) {
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
return ["week", "month"]
} else {
return site.validIntervalsByPeriod[query.period]
}
}
function getStoredInterval(period, domain) {
export const getStoredInterval = function(period, domain) {
return storage.getItem(`interval__${period}__${domain}`)
}
function storeInterval(period, domain, interval) {
export const storeInterval = function(period, domain, interval) {
storage.setItem(`interval__${period}__${domain}`, interval)
}
@ -41,47 +33,37 @@ function subscribeKeybinding(element) {
}, [handleKeyPress])
}
export const getCurrentInterval = function(site, query) {
const options = validIntervals(site, query)
const storedInterval = getStoredInterval(query.period, site.domain)
const defaultInterval = [...options].pop()
if (storedInterval && options.includes(storedInterval)) {
return storedInterval
} else {
return defaultInterval
}
function DropdownItem({ option, currentInterval, updateInterval }) {
return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
}
export function IntervalPicker({ query, site, onIntervalUpdate }) {
export function IntervalPicker({ graphData, query, site, updateInterval }) {
if (query.period == 'realtime') return null
const menuElement = React.useRef(null)
const options = validIntervals(site, query)
const currentInterval = getCurrentInterval(site, query)
const menuElement = React.useRef(null)
subscribeKeybinding(menuElement)
function updateInterval(interval) {
storeInterval(query.period, site.domain, interval)
onIntervalUpdate(interval)
let currentInterval = graphData?.interval
let options = site.validIntervalsByPeriod[query.period]
if (query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
options = ["week", "month"]
}
function renderDropdownItem(option) {
return (
<Menu.Item onClick={() => updateInterval(option)} key={option} disabled={option == currentInterval}>
{({ active }) => (
<span className={classNames({
'bg-gray-100 dark:bg-gray-900 text-gray-900 dark:text-gray-200 cursor-pointer': active,
'text-gray-700 dark:text-gray-200': !active,
'font-bold cursor-none select-none': option == currentInterval,
}, 'block px-4 py-2 text-sm')}>
{INTERVAL_LABELS[option]}
</span>
)}
</Menu.Item>
)
if (!options.includes(currentInterval)) {
currentInterval = [...options].pop()
}
return (
@ -103,7 +85,7 @@ export function IntervalPicker({ query, site, onIntervalUpdate }) {
leaveFrom="opacity-100 scale-100"
leaveTo="opacity-0 scale-95">
<Menu.Items className="py-1 text-left origin-top-right absolute right-0 mt-2 w-56 rounded-md shadow-lg bg-white dark:bg-gray-800 ring-1 ring-black ring-opacity-5 focus:outline-none z-10" static>
{options.map(renderDropdownItem)}
{options.map((option) => DropdownItem({ option, currentInterval, updateInterval }))}
</Menu.Items>
</Transition>
</>

@ -1,247 +0,0 @@
import React from 'react';
import { withRouter } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
import dateFormatter from './date-formatter';
import FadeIn from '../../fade-in';
import classNames from 'classnames';
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
}
getGraphMetric() {
let metric = this.props.graphData.metric
if (metric == 'visitors' && this.props.query.filters.goal) {
return 'conversions'
} else {
return metric
}
}
regenerateChart() {
const { graphData, query } = this.props
const metric = this.getGraphMetric()
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.maybeHopToHoveredPeriod.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
window.addEventListener('mousemove', this.repositionTooltip);
}
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
}
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
maybeHopToHoveredPeriod(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
render() {
const { graphData } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<FadeIn show={graphData}>
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
)
}
}
export default withRouter(LineGraph)

@ -1,17 +0,0 @@
import React from "react"
export default function SamplingNotice({topStatData}) {
const samplePercent = topStatData?.samplePercent
if (samplePercent && samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
} else {
return null
}
}

@ -1,53 +0,0 @@
import React, { useState } from "react"
import * as api from '../../api'
import { getCurrentInterval } from "./interval-picker"
export default function StatsExport({site, query}) {
const [exporting, setExporting] = useState(false)
function startExport() {
setExporting(true)
document.cookie = "exporting="
pollExportReady()
}
function pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(pollExportReady, 1000)
} else {
setExporting(false)
}
}
function renderLoading() {
return (
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
)
}
function renderExportLink() {
const interval = getCurrentInterval(site, query)
const queryParams = api.serializeQuery(query, [{ interval }])
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
return (
<a href={endpoint} download onClick={startExport}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
<polyline points="7 10 12 15 17 10"></polyline>
<line x1="12" y1="15" x2="12" y2="3"></line>
</svg>
</a>
)
}
return (
<div className="w-4 h-4 mx-2">
{exporting && renderLoading()}
{!exporting && renderExportLink()}
</div>
)
}

@ -3,9 +3,8 @@ import { Tooltip } from '../../util/tooltip'
import { SecondsSinceLastLoad } from '../../util/seconds-since-last-load'
import classNames from "classnames";
import numberFormatter, { durationFormatter } from '../../util/number-formatter'
import * as storage from '../../util/storage'
import { METRIC_MAPPING } from './graph-util'
import { formatDateRange } from '../../util/date.js'
import { getGraphableMetrics } from "./graph-util.js";
function Maybe({condition, children}) {
if (condition) {
@ -15,103 +14,103 @@ function Maybe({condition, children}) {
}
}
function renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg,
"pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
"pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
} else if (comparison < 0) {
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
} else if (comparison === 0) {
return <span className={noChangeClassName}>&#12336; 0%</span>
} else {
return null
export default class TopStats extends React.Component {
renderPercentageComparison(name, comparison, forceDarkBg = false) {
const formattedComparison = numberFormatter(Math.abs(comparison))
const defaultClassName = classNames({
"pl-2 text-xs dark:text-gray-100": !forceDarkBg,
"pl-2 text-xs text-gray-100": forceDarkBg
})
const noChangeClassName = classNames({
"pl-2 text-xs text-gray-700 dark:text-gray-300": !forceDarkBg,
"pl-2 text-xs text-gray-300": forceDarkBg
})
if (comparison > 0) {
const color = name === 'Bounce rate' ? 'text-red-400' : 'text-green-500'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&uarr;</span> {formattedComparison}%</span>
} else if (comparison < 0) {
const color = name === 'Bounce rate' ? 'text-green-500' : 'text-red-400'
return <span className={defaultClassName}><span className={color + ' font-bold'}>&darr;</span> {formattedComparison}%</span>
} else if (comparison === 0) {
return <span className={noChangeClassName}>&#12336; 0%</span>
} else {
return null
}
}
}
function topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short
} else {
return numberFormatter(value)
topStatNumberShort(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.short
} else {
return numberFormatter(value)
}
}
}
function topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long
} else {
return (value || 0).toLocaleString()
topStatNumberLong(name, value) {
if (['visit duration', 'time on page'].includes(name.toLowerCase())) {
return durationFormatter(value)
} else if (['bounce rate', 'conversion rate'].includes(name.toLowerCase())) {
return value + '%'
} else if (['average revenue', 'total revenue'].includes(name.toLowerCase())) {
return value?.long
} else {
return (value || 0).toLocaleString()
}
}
}
export default function TopStats(props) {
const {site, query, data, onMetricUpdate, tooltipBoundary, lastLoadTimestamp} = props
function tooltip(stat) {
topStatTooltip(stat, query) {
let statName = stat.name.toLowerCase()
statName = stat.value === 1 ? statName.slice(0, -1) : statName
const { topStatData, lastLoadTimestamp } = this.props
const showingImported = topStatData?.imported_source && topStatData?.with_imported
return (
<div>
{query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} vs. {topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{renderPercentageComparison(stat.name, stat.change, true)}</span>
{this.topStatNumberLong(stat.name, stat.value)} vs. {this.topStatNumberLong(stat.name, stat.comparison_value)} {statName}
<span className="ml-2">{this.renderPercentageComparison(stat.name, stat.change, true)}</span>
</div>}
{!query.comparison && <div className="whitespace-nowrap">
{topStatNumberLong(stat.name, stat.value)} {statName}
{this.topStatNumberLong(stat.name, stat.value)} {statName}
</div>}
{stat.name === 'Current visitors' && <p className="font-normal text-xs">Last updated <SecondsSinceLastLoad lastLoadTimestamp={lastLoadTimestamp}/>s ago</p>}
{stat.name === 'Views per visit' && showingImported && <p className="font-normal text-xs whitespace-nowrap">Based only on native data</p>}
</div>
)
}
function canMetricBeGraphed(stat) {
const graphableMetrics = getGraphableMetrics(query, site)
return stat.graph_metric && graphableMetrics.includes(stat.graph_metric)
canMetricBeGraphed(stat) {
const isTotalUniqueVisitors = this.props.query.filters.goal && stat.name === 'Unique visitors'
const isKnownMetric = Object.keys(METRIC_MAPPING).includes(stat.name)
return isKnownMetric && !isTotalUniqueVisitors
}
function maybeUpdateMetric(stat) {
if (canMetricBeGraphed(stat)) {
storage.setItem(`metric__${site.domain}`, stat.graph_metric)
onMetricUpdate(stat.graph_metric)
maybeUpdateMetric(stat) {
if (this.canMetricBeGraphed(stat)) {
this.props.updateMetric(METRIC_MAPPING[stat.name])
}
}
function blinkingDot() {
blinkingDot() {
return (
<div key="dot" className="block pulsating-circle" style={{ left: '125px', top: '52px' }}></div>
)
}
function getStoredMetric() {
return storage.getItem(`metric__${site.domain}`)
}
function renderStatName(stat) {
const isSelected = stat.graph_metric === getStoredMetric()
renderStatName(stat) {
const { metric } = this.props
const isSelected = metric === METRIC_MAPPING[stat.name]
const [statDisplayName, statExtraName] = stat.name.split(/(\(.+\))/g)
@ -128,45 +127,48 @@ export default function TopStats(props) {
)
}
function renderStat(stat, index) {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
render() {
const { topStatData, query, site } = this.props
const stats = topStatData && topStatData.top_stats.map((stat, index) => {
const className = classNames('px-4 md:px-6 w-1/2 my-4 lg:w-auto group select-none', {
'cursor-pointer': this.canMetricBeGraphed(stat),
'lg:border-l border-gray-300': index > 0,
'border-r lg:border-r-0': index % 2 === 0
})
return (
<Tooltip key={stat.name} info={this.topStatTooltip(stat, query)} className={className} onClick={() => { this.maybeUpdateMetric(stat) }} boundary={this.props.tooltipBoundary}>
{this.renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={METRIC_MAPPING[stat.name]}>{this.topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ this.renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, topStatData.from, topStatData.to) }</p>
</Maybe>
</div>
return (
<Tooltip key={stat.name} info={tooltip(stat, query)} className={className} onClick={() => { maybeUpdateMetric(stat) }} boundary={tooltipBoundary}>
{renderStatName(stat)}
<div className="my-1 space-y-2">
<div>
<span className="flex items-center justify-between whitespace-nowrap">
<p className="font-bold text-xl dark:text-gray-100" id={stat.graph_metric}>{topStatNumberShort(stat.name, stat.value)}</p>
<Maybe condition={!query.comparison}>
{ renderPercentageComparison(stat.name, stat.change) }
</Maybe>
</span>
<Maybe condition={query.comparison}>
<p className="text-xs dark:text-gray-100">{ formatDateRange(site, data.from, data.to) }</p>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ this.topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, topStatData.comparing_from, topStatData.comparing_to) }</p>
</div>
</Maybe>
</div>
<Maybe condition={query.comparison}>
<div>
<p className="font-bold text-xl text-gray-500 dark:text-gray-400">{ topStatNumberShort(stat.name, stat.comparison_value) }</p>
<p className="text-xs text-gray-500 dark:text-gray-400">{ formatDateRange(site, data.comparing_from, data.comparing_to) }</p>
</div>
</Maybe>
</div>
</Tooltip>
)
}
</Tooltip>
)
})
const stats = data && data.top_stats.map(renderStat)
if (stats && query && query.period === 'realtime') {
stats.push(this.blinkingDot())
}
if (stats && query.period === 'realtime') {
stats.push(blinkingDot())
return stats || null;
}
return stats || null;
}

@ -1,153 +1,534 @@
import React, { useState, useEffect, useRef, useCallback } from 'react';
import React from 'react';
import { withRouter, Link } from 'react-router-dom'
import Chart from 'chart.js/auto';
import { navigateToQuery } from '../../query'
import * as api from '../../api'
import * as storage from '../../util/storage'
import { getGraphableMetrics } from './graph-util'
import LazyLoader from '../../components/lazy-loader'
import GraphTooltip from './graph-tooltip'
import { buildDataSet, METRIC_MAPPING, METRIC_LABELS, METRIC_FORMATTER, LoadingState } from './graph-util'
import dateFormatter from './date-formatter';
import TopStats from './top-stats';
import { IntervalPicker, getCurrentInterval } from './interval-picker'
import StatsExport from './stats-export'
import WithImportedSwitch from './with-imported-switch';
import SamplingNotice from './sampling-notice';
import { IntervalPicker, getStoredInterval, storeInterval } from './interval-picker';
import FadeIn from '../../fade-in';
import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input'
import LineGraphWithRouter from './line-graph'
function fetchTopStats(site, query) {
const q = { ...query }
if (!isComparisonEnabled(q.comparison)) {
q.comparison = 'previous_period'
}
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
.flatMap((item) => item.data)
.map((item) => item || 0)
return api.get(url.apiPath(site, '/top-stats'), q)
if (yAxisValues) {
return Math.max(...yAxisValues)
} else {
return 1
}
}
function fetchMainGraph(site, query, metric, interval) {
const params = {metric, interval}
return api.get(url.apiPath(site, '/main-graph'), query, params)
}
class LineGraph extends React.Component {
constructor(props) {
super(props);
this.boundary = React.createRef()
this.regenerateChart = this.regenerateChart.bind(this);
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
this.state = {
exported: false
};
}
export default function VisitorGraph(props) {
const {site, query, lastLoadTimestamp} = props
const isRealtime = query.period === 'realtime'
const isDarkTheme = document.querySelector('html').classList.contains('dark') || false
regenerateChart() {
const { graphData, metric, query } = this.props
const graphEl = document.getElementById("main-graph-canvas")
this.ctx = graphEl.getContext('2d');
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
const topStatsBoundary = useRef(null)
return new Chart(this.ctx, {
type: 'line',
data: {
labels: graphData.labels,
datasets: dataSet
},
options: {
animation: false,
plugins: {
legend: { display: false },
tooltip: {
enabled: false,
mode: 'index',
intersect: false,
position: 'average',
external: GraphTooltip(graphData, metric, query)
},
},
responsive: true,
maintainAspectRatio: false,
onResize: this.updateWindowDimensions,
elements: { line: { tension: 0 }, point: { radius: 0 } },
onClick: this.onClick.bind(this),
scale: {
ticks: { precision: 0, maxTicksLimit: 8 }
},
scales: {
y: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
ticks: {
callback: METRIC_FORMATTER[metric],
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
},
grid: {
zeroLineColor: 'transparent',
drawBorder: false,
}
},
yComparison: {
min: 0,
suggestedMax: calculateMaximumY(dataSet),
display: false,
grid: { display: false },
},
x: {
grid: { display: false },
ticks: {
callback: function(val, _index, _ticks) {
if (this.getLabelForValue(val) == "__blank__") return ""
const [topStatData, setTopStatData] = useState(null)
const [topStatsLoading, setTopStatsLoading] = useState(true)
const [graphData, setGraphData] = useState(null)
const [graphLoading, setGraphLoading] = useState(true)
const hasMultipleYears =
graphData.labels
.filter((date) => typeof date === 'string')
.map(date => date.split('-')[0])
.filter((value, index, list) => list.indexOf(value) === index)
.length > 1
// This state is explicitly meant for the situation where either graph interval
// or graph metric is changed. That results in behaviour where Top Stats stay
// intact, but the graph container alone will display a loading spinner for as
// long as new graph data is fetched.
const [graphRefreshing, setGraphRefreshing] = useState(false)
if (graphData.interval === 'hour' && query.period !== 'day') {
const date = dateFormatter({
interval: "date",
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const hour = dateFormatter({
interval: graphData.interval,
longForm: false,
period: query.period,
shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
const onIntervalUpdate = useCallback((newInterval) => {
setGraphData(null)
setGraphRefreshing(true)
fetchGraphData(getStoredMetric(), newInterval)
}, [query])
// Returns a combination of date and hour. This is because
// small intervals like hour may return multiple days
// depending on the query period.
return `${date}, ${hour}`
}
const onMetricUpdate = useCallback((newMetric) => {
setGraphData(null)
setGraphRefreshing(true)
fetchGraphData(newMetric, getCurrentInterval(site, query))
}, [query])
if (graphData.interval === 'minute' && query.period !== 'realtime') {
return dateFormatter({
interval: "hour", longForm: false, period: query.period,
})(this.getLabelForValue(val))
}
useEffect(() => {
setTopStatData(null)
setTopStatsLoading(true)
setGraphData(null)
setGraphLoading(true)
fetchTopStatsAndGraphData()
return dateFormatter({
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
})(this.getLabelForValue(val))
},
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
}
}
},
interaction: {
mode: 'index',
intersect: false,
}
}
});
}
if (isRealtime) {
document.addEventListener('tick', fetchTopStatsAndGraphData)
repositionTooltip(e) {
const tooltipEl = document.getElementById('chartjs-tooltip');
if (tooltipEl && window.innerWidth >= 768) {
if (e.clientX > 0.66 * window.innerWidth) {
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
tooltipEl.style.left = null;
} else {
tooltipEl.style.right = null;
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
}
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
tooltipEl.style.opacity = 1;
}
}
return () => {
document.removeEventListener('tick', fetchTopStatsAndGraphData)
componentDidMount() {
if (this.props.graphData) {
this.chart = this.regenerateChart();
}
}, [query])
window.addEventListener('mousemove', this.repositionTooltip);
}
useEffect(() => {
if (topStatData) { storeTopStatsContainerHeight() }
}, [topStatData])
componentDidUpdate(prevProps) {
const { graphData, darkTheme } = this.props;
const tooltip = document.getElementById('chartjs-tooltip');
function fetchTopStatsAndGraphData() {
fetchTopStats(site, query)
.then((res) => {
setTopStatData(res)
setTopStatsLoading(false)
})
let metric = getStoredMetric()
const availableMetrics = getGraphableMetrics(query, site)
if (!availableMetrics.includes(metric)) {
metric = availableMetrics[0]
storage.setItem(`metric__${site.domain}`, metric)
if (
graphData !== prevProps.graphData ||
darkTheme !== prevProps.darkTheme
) {
if (graphData) {
if (this.chart) {
this.chart.destroy();
}
this.chart = this.regenerateChart();
this.chart.update();
}
if (tooltip) {
tooltip.style.display = 'none';
}
}
const interval = getCurrentInterval(site, query)
if (!graphData) {
if (this.chart) {
this.chart.destroy();
}
fetchGraphData(metric, interval)
if (tooltip) {
tooltip.style.display = 'none';
}
}
}
function fetchGraphData(metric, interval) {
fetchMainGraph(site, query, metric, interval)
.then((res) => {
setGraphData(res)
setGraphLoading(false)
setGraphRefreshing(false)
})
componentWillUnmount() {
// Ensure that the tooltip doesn't hang around when we are loading more data
const tooltip = document.getElementById('chartjs-tooltip');
if (tooltip) {
tooltip.style.opacity = 0;
tooltip.style.display = 'none';
}
window.removeEventListener('mousemove', this.repositionTooltip)
}
/**
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
* @param {*} chart - The chart instance.
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
*/
updateWindowDimensions(chart, dimensions) {
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
}
onClick(e) {
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
if (this.props.graphData.interval === 'month') {
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
} else if (this.props.graphData.interval === 'date') {
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
}
}
pollExportReady() {
if (document.cookie.includes('exporting')) {
setTimeout(this.pollExportReady.bind(this), 1000);
} else {
this.setState({ exported: false })
}
}
downloadSpinner() {
this.setState({ exported: true });
document.cookie = "exporting=";
setTimeout(this.pollExportReady.bind(this), 1000);
}
downloadLink() {
if (this.props.query.period !== 'realtime') {
if (this.state.exported) {
return (
<div className="w-4 h-4 mx-2">
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
</div>
)
} else {
const interval = this.props.graphData?.interval
const queryParams = api.serializeQuery(this.props.query, [{ interval }])
const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${queryParams}`
return (
<a className="w-4 h-4 mx-2" href={endpoint} download onClick={this.downloadSpinner.bind(this)}>
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path><polyline points="7 10 12 15 17 10"></polyline><line x1="12" y1="15" x2="12" y2="3"></line></svg>
</a>
)
}
}
}
function getStoredMetric() {
return storage.getItem(`metric__${site.domain}`)
samplingNotice() {
const samplePercent = this.props.topStatData && this.props.topStatData.sample_percent
if (samplePercent < 100) {
return (
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
</div>
)
}
}
function storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${site.domain}`, document.getElementById('top-stats-container').clientHeight)
importedNotice() {
if (!this.props.topStatData?.imported_source) return
const isBeforeNativeStats = (date) => {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(this.props.site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(this.props.topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
const strike = withImported ? "" : " line-through"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
<svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
</svg>
</div>
</Link>
)
}
}
// This function is used for maintaining the main-graph/top-stats container height in the
// loading process. The container height depends on how many top stat metrics are returned
// from the API, but in the loading state, we don't know that yet. We can use localStorage
// to keep track of the Top Stats container height.
function getTopStatsHeight() {
if (topStatData) {
getTopStatsHeight() {
if (this.props.topStatData) {
return 'auto'
} else {
return `${storage.getItem(`topStatsHeight__${site.domain}`) || 89}px`
return `${storage.getItem(`topStatsHeight__${this.props.site.domain}`) || 89}px`
}
}
render() {
const { mainGraphRefreshing, updateMetric, updateInterval, metric, topStatData, query, site, graphData, lastLoadTimestamp } = this.props
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
return (
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{(topStatsLoading || graphLoading) && renderLoader()}
<FadeIn show={!(topStatsLoading || graphLoading)}>
<div id="top-stats-container" className="flex flex-wrap" ref={topStatsBoundary} style={{ height: getTopStatsHeight() }}>
<TopStats site={site} query={query} data={topStatData} onMetricUpdate={onMetricUpdate} tooltipBoundary={topStatsBoundary.current} lastLoadTimestamp={lastLoadTimestamp} />
return (
<div>
<div id="top-stats-container" className="flex flex-wrap" ref={this.boundary} style={{ height: this.getTopStatsHeight() }}>
<TopStats site={site} query={query} metric={metric} updateMetric={updateMetric} topStatData={topStatData} tooltipBoundary={this.boundary.current} lastLoadTimestamp={lastLoadTimestamp} />
</div>
<div className="relative px-2">
{graphRefreshing && renderLoader()}
{mainGraphRefreshing && renderLoader()}
<div className="absolute right-4 -top-8 py-1 flex items-center">
{!isRealtime && <StatsExport site={site} query={query} />}
<SamplingNotice samplePercent={topStatData}/>
<WithImportedSwitch site={site} topStatData={topStatData} />
<IntervalPicker site={site} query={query} onIntervalUpdate={onIntervalUpdate} />
{this.downloadLink()}
{this.samplingNotice()}
{this.importedNotice()}
<IntervalPicker site={site} query={query} graphData={graphData} metric={metric} updateInterval={updateInterval} />
</div>
<LineGraphWithRouter graphData={graphData} darkTheme={isDarkTheme} query={query} />
<FadeIn show={graphData}>
<div className="relative h-96 w-full z-0">
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
</div>
</FadeIn>
</div>
</div>
)
}
}
const LineGraphWithRouter = withRouter(LineGraph)
export default class VisitorGraph extends React.Component {
constructor(props) {
super(props)
this.state = {
topStatsLoadingState: LoadingState.loading,
mainGraphLoadingState: LoadingState.loading,
metric: storage.getItem(`metric__${this.props.site.domain}`) || 'visitors'
}
this.onVisible = this.onVisible.bind(this)
this.updateMetric = this.updateMetric.bind(this)
this.fetchTopStatData = this.fetchTopStatData.bind(this)
this.fetchGraphData = this.fetchGraphData.bind(this)
this.updateInterval = this.updateInterval.bind(this)
}
isIntervalValid(interval) {
const { query, site } = this.props
const validIntervals = site.validIntervalsByPeriod[query.period] || []
return validIntervals.includes(interval)
}
getIntervalFromStorage() {
const { query, site } = this.props
let interval = getStoredInterval(query.period, site.domain)
if (interval !== "week" && interval !== "month" && query.period === "custom" && monthsBetweenDates(query.from, query.to) > 12) {
interval = "month"
}
if (this.isIntervalValid(interval)) {
return interval
} else {
return null
}
}
updateInterval(interval) {
if (this.isIntervalValid(interval)) {
storeInterval(this.props.query.period, this.props.site.domain, interval)
this.setState({ mainGraphLoadingState: LoadingState.refreshing, graphData: null }, this.fetchGraphData)
}
}
onVisible() {
this.setState({ mainGraphLoadingState: LoadingState.loading }, this.fetchGraphData)
this.fetchTopStatData()
if (this.props.query.period === 'realtime') {
document.addEventListener('tick', this.fetchGraphData)
document.addEventListener('tick', this.fetchTopStatData)
}
}
componentDidUpdate(prevProps, prevState) {
const { metric } = this.state;
const { query } = this.props
if (query !== prevProps.query) {
this.setState({ mainGraphLoadingState: LoadingState.loading, topStatsLoadingState: LoadingState.loading, graphData: null, topStatData: null }, this.fetchGraphData)
this.fetchTopStatData()
}
if (metric !== prevState.metric) {
this.setState({ mainGraphLoadingState: LoadingState.refreshing }, this.fetchGraphData)
}
}
resetMetric() {
const { topStatData } = this.state
const { query, site } = this.props
const savedMetric = storage.getItem(`metric__${site.domain}`)
const selectableMetrics = topStatData && topStatData.top_stats.map(({ name }) => METRIC_MAPPING[name]).filter(name => name)
const canSelectSavedMetric = selectableMetrics && selectableMetrics.includes(savedMetric)
if (query.filters.goal) {
this.setState({ metric: 'conversions' })
} else if (canSelectSavedMetric) {
this.setState({ metric: savedMetric })
} else {
this.setState({ metric: 'visitors' })
}
}
componentWillUnmount() {
document.removeEventListener('tick', this.fetchGraphData)
document.removeEventListener('tick', this.fetchTopStatData)
}
storeTopStatsContainerHeight() {
storage.setItem(`topStatsHeight__${this.props.site.domain}`, document.getElementById('top-stats-container').clientHeight)
}
updateMetric(clickedMetric) {
if (this.state.metric == clickedMetric) return
storage.setItem(`metric__${this.props.site.domain}`, clickedMetric)
this.setState({ metric: clickedMetric, graphData: null })
}
fetchGraphData() {
const url = `/api/stats/${encodeURIComponent(this.props.site.domain)}/main-graph`
let params = { metric: this.state.metric }
const interval = this.getIntervalFromStorage()
if (interval) { params.interval = interval }
api.get(url, this.props.query, params)
.then((res) => {
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: res })
return res
})
.catch((err) => {
console.log(err)
this.setState({ mainGraphLoadingState: LoadingState.loaded, graphData: false })
})
}
fetchTopStatData() {
const query = { ...this.props.query }
if (!isComparisonEnabled(query.comparison)) query.comparison = 'previous_period'
api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/top-stats`, query)
.then((res) => {
this.setState({ topStatsLoadingState: LoadingState.loaded, topStatData: res }, () => {
this.storeTopStatsContainerHeight()
this.resetMetric()
})
return res
})
}
renderInner() {
const { query, site } = this.props;
const { graphData, metric, topStatData, topStatsLoadingState, mainGraphLoadingState } = this.state;
const theme = document.querySelector('html').classList.contains('dark') || false
const mainGraphRefreshing = (mainGraphLoadingState === LoadingState.refreshing)
const topStatAndGraphLoaded = !!(topStatData && graphData)
const shouldShow =
topStatsLoadingState === LoadingState.loaded &&
LoadingState.isLoadedOrRefreshing(mainGraphLoadingState) &&
(topStatData && mainGraphRefreshing || topStatAndGraphLoaded)
return (
<FadeIn show={shouldShow}>
<LineGraphWithRouter mainGraphRefreshing={mainGraphRefreshing} graphData={graphData} topStatData={topStatData} site={site} query={query} darkTheme={theme} metric={metric} updateMetric={this.updateMetric} updateInterval={this.updateInterval} lastLoadTimestamp={this.props.lastLoadTimestamp} />
</FadeIn>
</div>
)
)
}
render() {
const { mainGraphLoadingState, topStatsLoadingState } = this.state
const showLoader =
[mainGraphLoadingState, topStatsLoadingState].includes(LoadingState.loading) &&
mainGraphLoadingState !== LoadingState.refreshing
return (
<LazyLoader onVisible={this.onVisible}>
<div className={"relative w-full mt-2 bg-white rounded shadow-xl dark:bg-gray-825"}>
{showLoader && renderLoader()}
{this.renderInner()}
</div>
</LazyLoader>
)
}
}
function renderLoader() {

@ -1,40 +0,0 @@
import React from "react"
import { parseNaiveDate, isBefore } from '../../util/date'
import { Link } from 'react-router-dom'
import * as url from '../../util/url'
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
export default function WithImportedSwitch({site, topStatData}) {
if (!topStatData?.imports_exist) {
return null
}
function isBeforeNativeStats(date) {
if (!date) return false
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
const parsedDate = parseNaiveDate(date)
return isBefore(parsedDate, nativeStatsBegin, "day")
}
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const withImported = topStatData.with_imported;
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
<BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)
} else {
return null
}
}

@ -57,7 +57,7 @@ function Regions({query, site, onClick}) {
getFilterFor={getFilterFor}
onClick={onClick}
keyLabel="Region"
metrics={maybeWithCR([VISITORS_METRIC], query)}
metrics={[VISITORS_METRIC]}
detailsLink={sitePath(site, '/regions')}
query={query}
renderIcon={renderIcon}
@ -84,7 +84,7 @@ function Cities({query, site}) {
fetchData={fetchData}
getFilterFor={getFilterFor}
keyLabel="City"
metrics={maybeWithCR([VISITORS_METRIC], query)}
metrics={[VISITORS_METRIC]}
detailsLink={sitePath(site, '/cities')}
query={query}
renderIcon={renderIcon}

@ -1,111 +1,99 @@
import React, { useEffect, useState, useMemo } from 'react'
import React, { useCallback, useEffect, useState } from 'react'
import { withRouter } from "react-router-dom";
import { FILTER_OPERATIONS } from "../../util/filters";
import Combobox from '../../components/combobox'
import FilterTypeSelector from "../../components/filter-type-selector";
import { FILTER_TYPES } from "../../util/filters";
import { parseQuery } from '../../query'
import { siteBasePath } from '../../util/url'
import { toFilterQuery, parseQueryPropsFilter } from '../../util/filters';
import * as api from '../../api'
import { apiPath, siteBasePath } from '../../util/url'
import { toFilterQuery, parseQueryFilter } from '../../util/filters';
import { shouldIgnoreKeypress } from '../../keybinding';
import PropFilterRow from './prop-filter-row';
function getFormState(query) {
if (query.filters['props']) {
const values = Object.fromEntries(parseQueryPropsFilter(query, 'props').map((value, index) => [index, value]))
const rawValue = query.filters['props']
if (rawValue) {
const [[propKey, _propValue]] = Object.entries(rawValue)
const {type, clauses} = parseQueryFilter(query, 'props')
return {
entries: Object.keys(values).sort(),
values
}
} else {
return {
entries: [0],
values: {
0: {
propKey: null,
type: FILTER_OPERATIONS.is,
clauses: []
}
}
prop_key: {value: propKey, label: propKey},
prop_value: { type: type, clauses: clauses }
}
}
return {
prop_key: null,
prop_value: { type: FILTER_TYPES.is, clauses: [] }
}
}
function PropFilterModal(props) {
const query = parseQuery(props.location.search, props.site)
const [formState, setFormState] = useState(getFormState(query))
const selectedPropKeys = useMemo(() => Object.values(formState.values).map((value) => value.propKey), [formState])
function onPropKeySelect(id, selectedOptions) {
const newPropKey = selectedOptions.length === 0 ? null : selectedOptions[0]
setFormState(prevState => ({
...prevState,
values: {
...prevState.values,
[id]: {
...prevState.values[id],
propKey: newPropKey,
clauses: []
}
}
}))
function fetchPropKeyOptions() {
return (input) => {
return api.get(apiPath(props.site, "/suggestions/prop_key"), query, { q: input.trim() })
}
}
function onPropValueSelect(id, selection) {
setFormState(prevState => ({
...prevState,
values: {
...prevState.values,
[id]: {
...prevState.values[id],
clauses: selection
}
}
}))
const fetchPropValueOptions = useCallback(() => {
return (input) => {
const propKey = formState.prop_key?.value
const updatedQuery = { ...query, filters: { ...query.filters, props: {[propKey]: '!(none)'} } }
return api.get(apiPath(props.site, "/suggestions/prop_value"), updatedQuery, { q: input.trim() })
}
}, [formState.prop_key])
function onPropKeySelect() {
return (selectedOptions) => {
const newPropKey = selectedOptions.length === 0 ? null : selectedOptions[0]
setFormState(prevState => ({
prop_key: newPropKey,
prop_value: { type: prevState.prop_value.type, clauses: [] }
}))
}
}
function onFilterTypeSelect(id, newType) {
setFormState(prevState => ({
...prevState,
values: {
...prevState.values,
[id]: {
...prevState.values[id],
type: newType
}
}
}))
function onPropValueSelect() {
return (selection) => {
setFormState(prevState => ({
...prevState, prop_value: { ...prevState.prop_value, clauses: selection }
}))
}
}
function onPropAdd() {
setFormState(prevState => {
const id = prevState.entries[prevState.entries.length - 1] + 1
return {
entries: prevState.entries.concat([id]),
values: {
...prevState.values,
[id]: {
propKey: null,
type: FILTER_OPERATIONS.is,
clauses: []
}
}
}
})
function onFilterTypeSelect() {
return (newType) => {
setFormState(prevState => ({
...prevState, prop_value: { ...prevState.prop_value, type: newType }
}))
}
}
function onPropDelete(id) {
setFormState(prevState => {
const entries = prevState.entries.filter((entry) => entry != id)
const values = {...prevState.values}
delete values[id]
function selectedFilterType() {
return formState.prop_value.type
}
return { entries, values }
})
function renderFilterInputs() {
return (
<div className="grid grid-cols-11 mt-6">
<div className="col-span-4">
<Combobox className="mr-2" fetchOptions={fetchPropKeyOptions()} singleOption={true} values={formState.prop_key ? [formState.prop_key] : []} onSelect={onPropKeySelect()} placeholder={'Property'} />
</div>
<div className="col-span-3 mx-2">
<FilterTypeSelector isDisabled={!formState.prop_key} forFilter={'prop_value'} onSelect={onFilterTypeSelect()} selectedType={selectedFilterType()} />
</div>
<div className="col-span-4">
<Combobox isDisabled={!formState.prop_key} fetchOptions={fetchPropValueOptions()} values={formState.prop_value.clauses} onSelect={onPropValueSelect()} placeholder={'Value'} />
</div>
</div>
)
}
function isDisabled() {
return formState.entries.some((id) => !formState.values[id].propKey || formState.values[id].clauses.length == 0)
return !(formState.prop_key && formState.prop_value.clauses.length > 0)
}
function shouldShowClear() {
@ -113,11 +101,8 @@ function PropFilterModal(props) {
}
function handleSubmit() {
const formFilters = Object.fromEntries(
Object.entries(formState.values)
.map(([_, {propKey, type, clauses}]) => [propKey.value, toFilterQuery(type, clauses)])
)
selectFiltersAndCloseModal(JSON.stringify(formFilters))
const filterString = JSON.stringify({ [formState.prop_key.value]: toFilterQuery(formState.prop_value.type, formState.prop_value.clauses) })
selectFiltersAndCloseModal(filterString)
}
function selectFiltersAndCloseModal(filterString) {
@ -154,27 +139,7 @@ function PropFilterModal(props) {
<div className="mt-4 border-b border-gray-300"></div>
<main className="modal__content">
<form className="flex flex-col" onSubmit={handleSubmit}>
{formState.entries.map((id) => (
<PropFilterRow
key={id}
id={id}
site={props.site}
query={query}
{...formState.values[id]}
showDelete={formState.entries.length > 1}
selectedPropKeys={selectedPropKeys}
onPropKeySelect={onPropKeySelect}
onPropValueSelect={onPropValueSelect}
onFilterTypeSelect={onFilterTypeSelect}
onPropDelete={onPropDelete}
/>
))}
<div className="mt-6">
<a className="underline text-indigo-500 text-sm cursor-pointer" onClick={onPropAdd}>
+ Add another
</a>
</div>
{renderFilterInputs()}
<div className="mt-6 flex items-center justify-start">
<button
@ -189,7 +154,7 @@ function PropFilterModal(props) {
<button
type="button"
className="ml-2 button px-4 flex bg-red-500 dark:bg-red-500 hover:bg-red-600 dark:hover:bg-red-700 items-center"
onClick={() => { selectFiltersAndCloseModal(null) }}
onClick={() => {selectFiltersAndCloseModal(null)}}
>
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"></path></svg>
Remove filter
@ -202,4 +167,4 @@ function PropFilterModal(props) {
)
}
export default withRouter(PropFilterModal)
export default withRouter(PropFilterModal)

@ -1,86 +0,0 @@
import React from 'react'
import Combobox from '../../components/combobox'
import FilterTypeSelector from "../../components/filter-type-selector";
import { FILTER_OPERATIONS } from "../../util/filters";
import * as api from '../../api'
import { apiPath } from '../../util/url'
import { TrashIcon } from '@heroicons/react/20/solid'
function PropFilterRow({
id,
query,
site,
propKey,
type,
clauses,
showDelete,
selectedPropKeys,
onPropKeySelect,
onPropValueSelect,
onFilterTypeSelect,
onPropDelete
}) {
function fetchPropKeyOptions() {
return (input) => {
return api.get(apiPath(site, "/suggestions/prop_key"), query, { q: input.trim() })
}
}
function fetchPropValueOptions() {
return (input) => {
if (type === FILTER_OPERATIONS.contains) {
return Promise.resolve([])
}
const key = propKey?.value
const updatedQuery = { ...query, filters: { ...query.filters, props: { [key]: '!(none)' } } }
return api.get(apiPath(site, "/suggestions/prop_value"), updatedQuery, { q: input.trim() })
}
}
return (
<div className="grid grid-cols-12 mt-6">
<div className="col-span-4">
<Combobox
className="mr-2"
fetchOptions={fetchPropKeyOptions()}
singleOption
autoFocus
values={propKey ? [propKey] : []}
onSelect={(value) => onPropKeySelect(id, value)}
placeholder={'Property'}
disabledOptions={selectedPropKeys}
/>
</div>
<div className="col-span-3 mx-2">
<FilterTypeSelector
isDisabled={!propKey}
forFilter={'prop_value'}
onSelect={(value) => onFilterTypeSelect(id, value)}
selectedType={type}
/>
</div>
<div className="col-span-4">
<Combobox
isDisabled={!propKey}
fetchOptions={fetchPropValueOptions()}
values={clauses}
onSelect={(value) => onPropValueSelect(id, value)}
placeholder={'Value'}
freeChoice={type == FILTER_OPERATIONS.contains}
/>
</div>
{showDelete && (
<div className="col-span-1 flex flex-col justify-center">
<a className="ml-2 text-red-600 h-5 w-5 cursor-pointer" onClick={() => onPropDelete(id)}>
<TrashIcon />
</a>
</div>
)}
</div>
)
}
export default PropFilterRow

@ -3,7 +3,7 @@ import { withRouter } from 'react-router-dom'
import FilterTypeSelector from "../../components/filter-type-selector";
import Combobox from '../../components/combobox'
import { FILTER_GROUPS, parseQueryFilter, formatFilterGroup, formattedFilters, toFilterQuery, FILTER_OPERATIONS } from '../../util/filters'
import { FILTER_GROUPS, parseQueryFilter, formatFilterGroup, formattedFilters, toFilterQuery, FILTER_TYPES } from '../../util/filters'
import { parseQuery } from '../../query'
import * as api from '../../api'
import { apiPath, siteBasePath } from '../../util/url'
@ -32,7 +32,7 @@ class RegularFilterModal extends React.Component {
super(props)
const query = parseQuery(props.location.search, props.site)
const formState = getFormState(props.filterGroup, query)
this.handleKeydown = this.handleKeydown.bind(this)
this.state = { query, formState }
}
@ -92,7 +92,7 @@ class RegularFilterModal extends React.Component {
fetchOptions(filter) {
return (input) => {
const { query, formState } = this.state
if (formState[filter].type === FILTER_OPERATIONS.contains) {return Promise.resolve([])}
if (formState[filter].type === FILTER_TYPES.contains) {return Promise.resolve([])}
const formFilters = Object.fromEntries(
Object.entries(formState)

@ -20,24 +20,8 @@ class ModalTable extends React.Component {
.then((res) => this.setState({loading: false, list: res}))
}
showConversionRate() {
return !!this.state.query.filters.goal
}
showPercentage() {
return this.props.showPercentage && !this.showConversionRate()
}
label() {
if (this.state.query.period === 'realtime') {
return 'Current visitors'
}
if (this.showConversionRate()) {
return 'Conversions'
}
return 'Visitors'
return this.state.query.period === 'realtime' ? 'Current visitors' : 'Visitors'
}
renderTableItem(tableItem) {
@ -56,10 +40,11 @@ class ModalTable extends React.Component {
{tableItem.name}
</Link>
</td>
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.total_visitors)}</td>}
<td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.visitors)}</td>
{this.showPercentage() && <td className="p-2 w-32 font-medium" align="right">{tableItem.percentage}</td>}
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.conversion_rate)}%</td>}
<td className="p-2 w-32 font-medium" align="right">
{numberFormatter(tableItem.visitors)}
{tableItem.percentage >= 0 &&
<span className="inline-block text-xs w-8 pl-1 text-right">({tableItem.percentage}%)</span> }
</td>
</tr>
)
}
@ -81,11 +66,19 @@ class ModalTable extends React.Component {
<table className="w-max overflow-x-auto md:w-full table-striped table-fixed">
<thead>
<tr>
<th className="p-2 w-48 md:w-56 lg:w-1/3 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="left">{this.props.keyLabel}</th>
{this.showConversionRate() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right" >Total Visitors</th>}
<th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">{this.label()}</th>
{this.showPercentage() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">%</th>}
{this.showConversionRate() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">CR</th>}
<th
className="p-2 w-48 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400"
align="left"
>
{this.props.keyLabel}
</th>
<th
// eslint-disable-next-line max-len
className="p-2 w-32 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400"
align="right"
>
{this.label()}
</th>
</tr>
</thead>
<tbody>

@ -2,13 +2,11 @@ import React from 'react';
import SearchTerms from './search-terms'
import SourceList from './source-list'
import ReferrerList from './referrer-list'
import { isFilteringOnFixedValue } from '../../util/filters'
export default function Sources(props) {
if (props.query.filters.source === 'Google') {
return <SearchTerms {...props} />
} else if (isFilteringOnFixedValue(props.query, 'source')) {
} else if (props.query.filters.source) {
return <ReferrerList {...props} />
} else {
return <SourceList {...props} />

@ -7,7 +7,6 @@ import ListReport from '../reports/list'
import { VISITORS_METRIC, maybeWithCR } from '../reports/metrics';
import { Menu, Transition } from '@headlessui/react'
import { ChevronDownIcon } from '@heroicons/react/20/solid'
import { escapeFilterValue } from '../../util/filters'
import classNames from 'classnames'
const UTM_TAGS = {
@ -26,7 +25,7 @@ function AllSources(props) {
}
function getFilterFor(listItem) {
return { source: escapeFilterValue(listItem['name']) }
return { source: listItem['name'] }
}
function renderIcon(listItem) {
@ -61,7 +60,7 @@ function UTMSources(props) {
}
function getFilterFor(listItem) {
return { [props.tab]: escapeFilterValue(listItem['name']) }
return { [props.tab]: listItem['name'] }
}
return (

@ -1,5 +1,3 @@
const flags = JSON.parse(document.getElementById('stats-react-container').dataset.flags)
export const FILTER_GROUPS = {
'page': ['page', 'entry_page', 'exit_page'],
'source': ['source', 'referrer'],
@ -9,23 +7,19 @@ export const FILTER_GROUPS = {
'os': ['os', 'os_version'],
'utm': ['utm_medium', 'utm_source', 'utm_campaign', 'utm_term', 'utm_content'],
'goal': ['goal'],
'props': ['prop_key', 'prop_value'],
...(flags.hostname_filter ? { 'hostname': ['hostname'] } : {})
'props': ['prop_key', 'prop_value']
}
export const NO_CONTAINS_OPERATOR = new Set(['goal', 'screen'].concat(FILTER_GROUPS['location']))
export const FILTER_OPERATIONS = {
export const FILTER_TYPES = {
isNot: 'is not',
contains: 'contains',
is: 'is'
};
export const OPERATION_PREFIX = {
[FILTER_OPERATIONS.isNot]: '!',
[FILTER_OPERATIONS.contains]: '~',
[FILTER_OPERATIONS.is]: ''
export const FILTER_PREFIXES = {
[FILTER_TYPES.isNot]: '!',
[FILTER_TYPES.contains]: '~',
[FILTER_TYPES.is]: ''
};
export function supportsIsNot(filterName) {
@ -33,7 +27,7 @@ export function supportsIsNot(filterName) {
}
export function isFreeChoiceFilter(filterName) {
return !NO_CONTAINS_OPERATOR.has(filterName)
return FILTER_GROUPS['page'].concat(FILTER_GROUPS['utm']).includes(filterName)
}
// As of March 2023, Safari does not support negative lookbehind regexes. In case it throws an error, falls back to plain | matching. This means
@ -41,7 +35,7 @@ export function isFreeChoiceFilter(filterName) {
let NON_ESCAPED_PIPE_REGEX;
try {
NON_ESCAPED_PIPE_REGEX = new RegExp("(?<!\\\\)\\|", "g")
} catch (_e) {
} catch(_e) {
NON_ESCAPED_PIPE_REGEX = '|'
}
@ -52,61 +46,56 @@ export function escapeFilterValue(value) {
}
export function toFilterQuery(type, clauses) {
const prefix = OPERATION_PREFIX[type];
const prefix = FILTER_PREFIXES[type];
const result = clauses.map(clause => escapeFilterValue(clause.value.trim())).join('|')
return prefix + result;
}
export function parsePrefix(rawValue) {
const type = Object.keys(OPERATION_PREFIX)
.find(type => OPERATION_PREFIX[type] === rawValue[0]) || FILTER_OPERATIONS.is;
const type = Object.keys(FILTER_PREFIXES)
.find(type => FILTER_PREFIXES[type] === rawValue[0]) || FILTER_TYPES.is;
const value = type === FILTER_OPERATIONS.is ? rawValue : rawValue.substring(1)
const value = type === FILTER_TYPES.is ? rawValue : rawValue.substring(1)
const values = value
.split(NON_ESCAPED_PIPE_REGEX)
.filter((clause) => !!clause)
.map((val) => val.replaceAll(ESCAPED_PIPE, '|'))
return { type, values }
}
export function parseQueryPropsFilter(query) {
return Object.entries(query.filters['props']).map(([key, propVal]) => {
const { type, values } = parsePrefix(propVal)
const clauses = values.map(val => { return { value: val, label: val } })
return { propKey: { label: key, value: key }, type, clauses }
})
return {type, values}
}
export function parseQueryFilter(query, filter) {
const { type, values } = parsePrefix(query.filters[filter] || '')
let labels = values
if (filter === 'props') {
const rawValue = query.filters['props']
const [[_propKey, propVal]] = Object.entries(rawValue)
const {type, values} = parsePrefix(propVal)
const clauses = values.map(val => { return {value: val, label: val}})
return {type, clauses}
} else {
const {type, values} = parsePrefix(query.filters[filter] || '')
if (filter === 'country' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('country_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
let labels = values
if (filter === 'region' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('region_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
if (filter === 'country' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('country_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
if (filter === 'city' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('city_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
if (filter === 'region' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('region_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
const clauses = values.map((value, index) => { return { value, label: labels[index] } })
if (filter === 'city' && values.length > 0) {
const rawLabel = (new URLSearchParams(window.location.search)).get('city_labels') || ''
labels = rawLabel.split('|').filter(label => !!label)
}
return { type, clauses }
}
const clauses = values.map((value, index) => { return {value, label: labels[index]}})
export function isFilteringOnFixedValue(query, filter) {
const { type, clauses } = parseQueryFilter(query, filter)
return type == FILTER_OPERATIONS.is && clauses.length == 1
return {type, clauses}
}
}
export function formatFilterGroup(filterGroup) {
@ -156,7 +145,6 @@ export const formattedFilters = {
'region': 'Region',
'city': 'City',
'page': 'Page',
'hostname': 'Hostname',
'entry_page': 'Entry Page',
'exit_page': 'Exit Page',
'exit_page': 'Exit Page'
}

@ -5,7 +5,6 @@ export default (id) => ({
isOpen: false,
id: id,
focus: null,
selectionInProgress: false,
setFocus(f) {
this.focus = f;
},
@ -14,14 +13,9 @@ export default (id) => ({
this.setFocus(this.leastFocusableIndex())
}
},
trackSubmitValueChange() {
this.selectionInProgress = false;
},
open() {
if (!this.isOpen) {
this.initFocus()
this.isOpen = true
}
this.initFocus()
this.isOpen = true
},
suggestionsCount() {
return this.$refs.suggestions?.querySelectorAll('li').length
@ -67,7 +61,7 @@ export default (id) => ({
focusNext() {
const nextIndex = this.nextFocusableIndex()
this.open()
if (!this.isOpen) this.open()
this.setFocus(nextIndex)
this.scrollTo(nextIndex)
@ -75,7 +69,7 @@ export default (id) => ({
focusPrev() {
const prevIndex = this.prevFocusableIndex()
this.open()
if (!this.isOpen) this.open()
this.setFocus(prevIndex)
this.scrollTo(prevIndex)

@ -19,30 +19,12 @@ if (csrfToken && websocketUrl) {
})
}
}
let Uploaders = {}
Uploaders.S3 = function (entries, onViewError) {
entries.forEach(entry => {
let xhr = new XMLHttpRequest()
onViewError(() => xhr.abort())
xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
xhr.onerror = () => entry.error()
xhr.upload.addEventListener("progress", (event) => {
if (event.lengthComputable) {
let percent = Math.round((event.loaded / event.total) * 100)
if (percent < 100) { entry.progress(percent) }
}
})
let url = entry.meta.url
xhr.open("PUT", url, true)
xhr.send(entry.file)
})
}
let token = csrfToken.getAttribute("content")
let url = websocketUrl.getAttribute("content")
let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
let liveSocket = new LiveSocket(liveUrl, Socket, {
heartbeatIntervalMs: 10000,
params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {
params: { _csrf_token: token }, hooks: Hooks, dom: {
// for alpinejs integration
onBeforeElUpdated(from, to) {
if (from._x_dataStack) {

@ -8,7 +8,7 @@ TOTP_VAULT_KEY=Q3BD4nddbkVJIPXgHuo5NthGKSIH0yesRfG05J88HIo=
ENVIRONMENT=dev
MAILER_ADAPTER=Bamboo.LocalAdapter
LOG_LEVEL=debug
SELFHOST=true
SELFHOST=false
DISABLE_CRON=true
ADMIN_USER_IDS=1
SHOW_CITIES=true
@ -20,11 +20,3 @@ GOOGLE_CLIENT_SECRET=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
PROMEX_DISABLED=false
SITE_DEFAULT_INGEST_THRESHOLD=1000000
S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:10000
S3_EXPORTS_BUCKET=dev-exports
S3_IMPORTS_BUCKET=dev-imports

@ -15,11 +15,3 @@ IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb
SITE_DEFAULT_INGEST_THRESHOLD=1000000
GOOGLE_CLIENT_ID=fake_client_id
GOOGLE_CLIENT_SECRET=fake_client_secret
S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:10000
S3_EXPORTS_BUCKET=test-exports
S3_IMPORTS_BUCKET=test-imports

@ -38,14 +38,14 @@ config :tailwind,
config :ua_inspector,
database_path: "priv/ua_inspector",
remote_release: "6.3.0"
remote_release: "66d80de32fbb265941f4d7941fadc19097375097"
config :ref_inspector,
database_path: "priv/ref_inspector"
config :plausible,
paddle_api: Plausible.Billing.PaddleApi,
google_api: Plausible.Google.API
google_api: Plausible.Google.Api
config :plausible,
# 30 minutes
@ -65,7 +65,10 @@ config :plausible, Plausible.Repo,
connect_timeout: 300_000,
handshake_timeout: 300_000
config :plausible, Plausible.Cache, enabled: true
config :plausible,
sites_by_domain_cache_enabled: true,
sites_by_domain_cache_refresh_interval_max_jitter: :timer.seconds(5),
sites_by_domain_cache_refresh_interval: :timer.minutes(15)
config :plausible, Plausible.Ingestion.Counters, enabled: true
@ -73,8 +76,4 @@ config :ex_cldr,
default_locale: "en",
default_backend: Plausible.Cldr
config :sentry,
enable_source_code_context: true,
root_source_code_path: [File.cwd!()]
import_config "#{config_env()}.exs"

@ -229,7 +229,11 @@ ip_geolocation_db = get_var_from_path_or_env(config_dir, "IP_GEOLOCATION_DB", ge
geonames_source_file = get_var_from_path_or_env(config_dir, "GEONAMES_SOURCE_FILE")
maxmind_license_key = get_var_from_path_or_env(config_dir, "MAXMIND_LICENSE_KEY")
maxmind_edition = get_var_from_path_or_env(config_dir, "MAXMIND_EDITION", "GeoLite2-City")
persistent_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIR")
maxmind_cache_dir = get_var_from_path_or_env(config_dir, "PERSISTENT_CACHE_DIR")
if System.get_env("DISABLE_AUTH") do
Logger.warning("DISABLE_AUTH env var is no longer supported")
end
enable_email_verification =
config_dir
@ -287,16 +291,13 @@ secure_cookie =
|> get_var_from_path_or_env("SECURE_COOKIE", if(is_selfhost, do: "false", else: "true"))
|> String.to_existing_atom()
license_key = get_var_from_path_or_env(config_dir, "LICENSE_KEY", "")
config :plausible,
environment: env,
mailer_email: mailer_email,
super_admin_user_ids: super_admin_user_ids,
is_selfhost: is_selfhost,
custom_script_name: custom_script_name,
log_failed_login_attempts: log_failed_login_attempts,
license_key: license_key
log_failed_login_attempts: log_failed_login_attempts
config :plausible, :selfhost,
enable_email_verification: enable_email_verification,
@ -335,16 +336,21 @@ else
database: get_var_from_path_or_env(config_dir, "DATABASE_NAME", "plausible")
end
included_environments = if sentry_dsn, do: ["prod", "staging", "dev"], else: []
sentry_app_version = runtime_metadata[:version] || app_version
config :sentry,
dsn: sentry_dsn,
environment_name: env,
included_environments: included_environments,
release: sentry_app_version,
tags: %{app_version: sentry_app_version},
enable_source_code_context: true,
root_source_code_path: [File.cwd!()],
client: Plausible.Sentry.Client,
send_max_attempts: 1,
before_send: {Plausible.SentryFilter, :before_send}
filter: Plausible.SentryFilter,
before_send_event: {Plausible.SentryFilter, :before_send}
config :plausible, :paddle,
vendor_auth_code: paddle_auth_code,
@ -354,10 +360,8 @@ config :plausible, :google,
client_id: google_cid,
client_secret: google_secret,
api_url: "https://www.googleapis.com",
reporting_api_url: "https://analyticsreporting.googleapis.com"
config :plausible, :imported,
max_buffer_size: get_int_from_path_or_env(config_dir, "IMPORTED_MAX_BUFFER_SIZE", 10_000)
reporting_api_url: "https://analyticsreporting.googleapis.com",
max_buffer_size: get_int_from_path_or_env(config_dir, "GOOGLE_MAX_BUFFER_SIZE", 10_000)
maybe_ch_ipv6 =
get_var_from_path_or_env(config_dir, "ECTO_CH_IPV6", "false")
@ -384,8 +388,7 @@ config :plausible, Plausible.ClickhouseRepo,
url: ch_db_url,
transport_opts: ch_transport_opts,
settings: [
readonly: 1,
join_algorithm: "direct,parallel_hash"
readonly: 1
]
config :plausible, Plausible.IngestRepo,
@ -395,10 +398,7 @@ config :plausible, Plausible.IngestRepo,
transport_opts: ch_transport_opts,
flush_interval_ms: ch_flush_interval_ms,
max_buffer_size: ch_max_buffer_size,
pool_size: ingest_pool_size,
settings: [
materialized_views_ignore_errors: 1
]
pool_size: ingest_pool_size
config :plausible, Plausible.AsyncInsertRepo,
queue_target: 500,
@ -408,8 +408,7 @@ config :plausible, Plausible.AsyncInsertRepo,
pool_size: 1,
settings: [
async_insert: 1,
wait_for_async_insert: 0,
materialized_views_ignore_errors: 1
wait_for_async_insert: 0
]
config :plausible, Plausible.ImportDeletionRepo,
@ -467,20 +466,6 @@ case mailer_adapter do
retries: get_var_from_path_or_env(config_dir, "SMTP_RETRIES") || 2,
no_mx_lookups: get_var_from_path_or_env(config_dir, "SMTP_MX_LOOKUPS_ENABLED") || true
"Bamboo.Mua" ->
config :plausible, Plausible.Mailer, adapter: Bamboo.Mua
if relay = get_var_from_path_or_env(config_dir, "SMTP_HOST_ADDR") do
port = get_int_from_path_or_env(config_dir, "SMTP_HOST_PORT", 25)
username = get_var_from_path_or_env(config_dir, "SMTP_USER_NAME")
password = get_var_from_path_or_env(config_dir, "SMTP_USER_PWD")
config :plausible, Plausible.Mailer,
auth: [username: username, password: password],
relay: relay,
port: port
end
"Bamboo.LocalAdapter" ->
config :plausible, Plausible.Mailer, adapter: Bamboo.LocalAdapter
@ -522,8 +507,8 @@ cloud_cron = [
{"0 15 * * *", Plausible.Workers.NotifyAnnualRenewal},
# Every midnight
{"0 0 * * *", Plausible.Workers.LockSites},
# Daily at 8
{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil}
# Daily at 5
{"0 5 * * *", Plausible.Workers.AcceptTrafficUntil}
]
crontab = if(is_selfhost, do: base_cron, else: base_cron ++ cloud_cron)
@ -536,11 +521,9 @@ base_queues = [
check_stats_emails: 1,
site_setup_emails: 1,
clean_invitations: 1,
analytics_imports: 1,
google_analytics_imports: 1,
domain_change_transition: 1,
check_accept_traffic_until: 1,
# NOTE: maybe move s3_csv_export to cloud_queues?
s3_csv_export: 1
check_accept_traffic_until: 1
]
cloud_queues = [
@ -630,7 +613,7 @@ geo_opts =
[
license_key: maxmind_license_key,
edition: maxmind_edition,
cache_dir: persistent_cache_dir,
cache_dir: maxmind_cache_dir,
async: true
]
@ -682,7 +665,9 @@ else
traces_exporter: :none
end
config :tzdata, :data_dir, Path.join(persistent_cache_dir || System.tmp_dir!(), "tzdata_data")
config :tzdata,
:data_dir,
get_var_from_path_or_env(config_dir, "STORAGE_DIR", Application.app_dir(:tzdata, "priv"))
promex_disabled? =
config_dir
@ -709,73 +694,3 @@ if not is_selfhost do
config :plausible, Plausible.Site, default_ingest_threshold: site_default_ingest_threshold
end
s3_disabled? =
config_dir
|> get_var_from_path_or_env("S3_DISABLED", "true")
|> String.to_existing_atom()
unless s3_disabled? do
s3_env = [
%{
name: "S3_ACCESS_KEY_ID",
example: "AKIAIOSFODNN7EXAMPLE"
},
%{
name: "S3_SECRET_ACCESS_KEY",
example: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
},
%{
name: "S3_REGION",
example: "us-east-1"
},
%{
name: "S3_ENDPOINT",
example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com"
},
%{
name: "S3_EXPORTS_BUCKET",
example: "my-csv-exports-bucket"
},
%{
name: "S3_IMPORTS_BUCKET",
example: "my-csv-imports-bucket"
}
]
s3_env =
Enum.map(s3_env, fn var ->
Map.put(var, :value, get_var_from_path_or_env(config_dir, var.name))
end)
s3_missing_env = Enum.filter(s3_env, &is_nil(&1.value))
unless s3_missing_env == [] do
raise ArgumentError, """
Missing S3 configuration. Please set #{s3_missing_env |> Enum.map(& &1.name) |> Enum.join(", ")} environment variable(s):
#{s3_missing_env |> Enum.map(fn %{name: name, example: example} -> "\t#{name}=#{example}" end) |> Enum.join("\n")}
"""
end
s3_env_value = fn name ->
s3_env |> Enum.find(&(&1.name == name)) |> Map.fetch!(:value)
end
config :ex_aws,
http_client: Plausible.S3.Client,
access_key_id: s3_env_value.("S3_ACCESS_KEY_ID"),
secret_access_key: s3_env_value.("S3_SECRET_ACCESS_KEY"),
region: s3_env_value.("S3_REGION")
%URI{scheme: s3_scheme, host: s3_host, port: s3_port} = URI.parse(s3_env_value.("S3_ENDPOINT"))
config :ex_aws, :s3,
scheme: s3_scheme <> "://",
host: s3_host,
port: s3_port
config :plausible, Plausible.S3,
exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET"),
imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")
end

@ -16,15 +16,14 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter
config :plausible,
paddle_api: Plausible.PaddleApi.Mock,
google_api: Plausible.Google.API.Mock
google_api: Plausible.Google.Api.Mock
config :bamboo, :refute_timeout, 10
config :plausible,
session_timeout: 0,
http_impl: Plausible.HTTPClient.Mock
config :plausible, Plausible.Cache, enabled: false
http_impl: Plausible.HTTPClient.Mock,
sites_by_domain_cache_enabled: false
config :ex_money, api_module: Plausible.ExchangeRateMock

@ -1,5 +0,0 @@
Plausible copyright notice
This software is Copyright 2024 Plausible Insights OÜ
This repository is provided for informational purposes only, and no rights to use, distribute or otherwise exploit this software are granted to you. Our other repositories include software that is offered under the Affero GPL, but this repository is not. If you wish to discuss a license, or our hosted service that uses the software, please contact us at hello@plausible.io

@ -1,42 +0,0 @@
defmodule Plausible.License do
@moduledoc """
This module ensures that you cannot run Plausible Analytics Enterprise Edition without a valid license key.
The software contained within the ee/ and assets/js/dashboard/ee directories are Copyright © Plausible Insights OÜ.
We have made this code available solely for informational and transparency purposes. No rights are granted to use,
distribute, or exploit this software in any form.
Any attempt to disable or modify the behavior of this module will be considered a violation of copyright.
If you wish to use the Plausible Analytics Enterprise Edition for your own requirements, please contact us
at hello@plausible.io to discuss obtaining a license.
"""
require Logger
if Mix.env() == :prod do
def ensure_valid_license do
if has_valid_license?() do
:ok
else
Logger.error(
"Invalid or no license key provided for Plausible Enterprise Edition. Please contact hello@plausible.io to acquire a license."
)
Logger.error("Shutting down")
System.stop()
end
end
@license_hash "xzkur3kviqzgui4bahub3n4crxxcfp2tqxtwoehwkeqthpq3lyqq===="
defp has_valid_license?() do
hash =
:crypto.hash(:sha256, Application.fetch_env!(:plausible, :license_key))
|> Base.encode32(case: :lower)
hash == @license_hash
end
else
def ensure_valid_license do
:ok
end
end
end

@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics
end
def total_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
def total_revenue_query(query) do
from(e in query,
select_merge: %{
total_revenue:
fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end
def average_revenue_query() do
dynamic(
[e],
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
def average_revenue_query(query) do
from(e in query,
select_merge: %{
average_revenue:
fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
}
)
end

@ -84,11 +84,6 @@ defmodule PlausibleWeb.Live.FunnelSettings.Form do
submit_name={"funnel[steps][#{step_idx}][goal_id]"}
module={PlausibleWeb.Live.Components.ComboBox}
suggest_fun={&PlausibleWeb.Live.Components.ComboBox.StaticSearch.suggest/2}
on_selection_made={
fn value, by_id ->
send(self(), {:selection_made, %{submit_value: value, by: by_id}})
end
}
id={"step-#{step_idx}"}
options={reject_already_selected("step-#{step_idx}", @goals, @selections_made)}
/>

@ -1,38 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240302"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

@ -1,12 +0,0 @@
{
"account": "accounts/28425555",
"createTime": "2024-02-22T10:50:15.462Z",
"currencyCode": "USD",
"displayName": "account.one - GA4",
"name": "properties/428685444",
"parent": "accounts/28425555",
"propertyType": "PROPERTY_TYPE_ORDINARY",
"serviceLevel": "GOOGLE_ANALYTICS_STANDARD",
"timeZone": "Europe/Warsaw",
"updateTime": "2024-02-22T10:50:15.462Z"
}

@ -1,41 +0,0 @@
{
"accountSummaries": [
{
"account": "accounts/28425178",
"displayName": "account.one",
"name": "accountSummaries/28425178",
"propertySummaries": [
{
"displayName": "account.one - GA4",
"parent": "accounts/28425178",
"property": "properties/428685906",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
},
{
"account": "accounts/45336102",
"displayName": "account.two",
"name": "accountSummaries/45336102"
},
{
"account": "accounts/54516992",
"displayName": "Demo Account",
"name": "accountSummaries/54516992",
"propertySummaries": [
{
"displayName": "GA4 - Flood-It!",
"parent": "accounts/54516992",
"property": "properties/153293282",
"propertyType": "PROPERTY_TYPE_ORDINARY"
},
{
"displayName": "GA4 - Google Merch Shop",
"parent": "accounts/54516992",
"property": "properties/213025502",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
}
]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,786 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Amsterdam"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 31,
"rows": [
{
"dimensionValues": [
{
"value": "20240131"
}
],
"metricValues": [
{
"value": "141"
},
{
"value": "195"
},
{
"value": "56"
},
{
"value": "146"
},
{
"value": "4815"
}
]
},
{
"dimensionValues": [
{
"value": "20240130"
}
],
"metricValues": [
{
"value": "115"
},
{
"value": "171"
},
{
"value": "42"
},
{
"value": "115"
},
{
"value": "3481"
}
]
},
{
"dimensionValues": [
{
"value": "20240129"
}
],
"metricValues": [
{
"value": "105"
},
{
"value": "153"
},
{
"value": "54"
},
{
"value": "113"
},
{
"value": "2895"
}
]
},
{
"dimensionValues": [
{
"value": "20240128"
}
],
"metricValues": [
{
"value": "184"
},
{
"value": "259"
},
{
"value": "74"
},
{
"value": "190"
},
{
"value": "6861"
}
]
},
{
"dimensionValues": [
{
"value": "20240127"
}
],
"metricValues": [
{
"value": "155"
},
{
"value": "202"
},
{
"value": "67"
},
{
"value": "158"
},
{
"value": "5259"
}
]
},
{
"dimensionValues": [
{
"value": "20240126"
}
],
"metricValues": [
{
"value": "132"
},
{
"value": "189"
},
{
"value": "57"
},
{
"value": "142"
},
{
"value": "5545"
}
]
},
{
"dimensionValues": [
{
"value": "20240125"
}
],
"metricValues": [
{
"value": "151"
},
{
"value": "229"
},
{
"value": "71"
},
{
"value": "155"
},
{
"value": "5365"
}
]
},
{
"dimensionValues": [
{
"value": "20240124"
}
],
"metricValues": [
{
"value": "137"
},
{
"value": "221"
},
{
"value": "64"
},
{
"value": "148"
},
{
"value": "4454"
}
]
},
{
"dimensionValues": [
{
"value": "20240123"
}
],
"metricValues": [
{
"value": "166"
},
{
"value": "267"
},
{
"value": "61"
},
{
"value": "171"
},
{
"value": "4850"
}
]
},
{
"dimensionValues": [
{
"value": "20240122"
}
],
"metricValues": [
{
"value": "161"
},
{
"value": "233"
},
{
"value": "73"
},
{
"value": "165"
},
{
"value": "4871"
}
]
},
{
"dimensionValues": [
{
"value": "20240121"
}
],
"metricValues": [
{
"value": "254"
},
{
"value": "397"
},
{
"value": "114"
},
{
"value": "267"
},
{
"value": "8845"
}
]
},
{
"dimensionValues": [
{
"value": "20240120"
}
],
"metricValues": [
{
"value": "214"
},
{
"value": "383"
},
{
"value": "100"
},
{
"value": "223"
},
{
"value": "6914"
}
]
},
{
"dimensionValues": [
{
"value": "20240119"
}
],
"metricValues": [
{
"value": "381"
},
{
"value": "5571"
},
{
"value": "100"
},
{
"value": "228"
},
{
"value": "98549"
}
]
},
{
"dimensionValues": [
{
"value": "20240118"
}
],
"metricValues": [
{
"value": "177"
},
{
"value": "278"
},
{
"value": "86"
},
{
"value": "186"
},
{
"value": "5553"
}
]
},
{
"dimensionValues": [
{
"value": "20240117"
}
],
"metricValues": [
{
"value": "204"
},
{
"value": "271"
},
{
"value": "97"
},
{
"value": "216"
},
{
"value": "6777"
}
]
},
{
"dimensionValues": [
{
"value": "20240116"
}
],
"metricValues": [
{
"value": "200"
},
{
"value": "280"
},
{
"value": "65"
},
{
"value": "203"
},
{
"value": "9381"
}
]
},
{
"dimensionValues": [
{
"value": "20240115"
}
],
"metricValues": [
{
"value": "255"
},
{
"value": "311"
},
{
"value": "86"
},
{
"value": "269"
},
{
"value": "12365"
}
]
},
{
"dimensionValues": [
{
"value": "20240114"
}
],
"metricValues": [
{
"value": "243"
},
{
"value": "302"
},
{
"value": "86"
},
{
"value": "257"
},
{
"value": "11073"
}
]
},
{
"dimensionValues": [
{
"value": "20240113"
}
],
"metricValues": [
{
"value": "231"
},
{
"value": "276"
},
{
"value": "92"
},
{
"value": "244"
},
{
"value": "10084"
}
]
},
{
"dimensionValues": [
{
"value": "20240112"
}
],
"metricValues": [
{
"value": "215"
},
{
"value": "265"
},
{
"value": "75"
},
{
"value": "225"
},
{
"value": "10195"
}
]
},
{
"dimensionValues": [
{
"value": "20240111"
}
],
"metricValues": [
{
"value": "183"
},
{
"value": "235"
},
{
"value": "64"
},
{
"value": "191"
},
{
"value": "8200"
}
]
},
{
"dimensionValues": [
{
"value": "20240110"
}
],
"metricValues": [
{
"value": "190"
},
{
"value": "244"
},
{
"value": "61"
},
{
"value": "203"
},
{
"value": "9056"
}
]
},
{
"dimensionValues": [
{
"value": "20240109"
}
],
"metricValues": [
{
"value": "201"
},
{
"value": "290"
},
{
"value": "63"
},
{
"value": "213"
},
{
"value": "11651"
}
]
},
{
"dimensionValues": [
{
"value": "20240108"
}
],
"metricValues": [
{
"value": "195"
},
{
"value": "235"
},
{
"value": "71"
},
{
"value": "209"
},
{
"value": "7946"
}
]
},
{
"dimensionValues": [
{
"value": "20240107"
}
],
"metricValues": [
{
"value": "205"
},
{
"value": "274"
},
{
"value": "80"
},
{
"value": "218"
},
{
"value": "10048"
}
]
},
{
"dimensionValues": [
{
"value": "20240106"
}
],
"metricValues": [
{
"value": "256"
},
{
"value": "303"
},
{
"value": "119"
},
{
"value": "281"
},
{
"value": "8440"
}
]
},
{
"dimensionValues": [
{
"value": "20240105"
}
],
"metricValues": [
{
"value": "226"
},
{
"value": "270"
},
{
"value": "83"
},
{
"value": "237"
},
{
"value": "9311"
}
]
},
{
"dimensionValues": [
{
"value": "20240104"
}
],
"metricValues": [
{
"value": "223"
},
{
"value": "274"
},
{
"value": "81"
},
{
"value": "236"
},
{
"value": "12051"
}
]
},
{
"dimensionValues": [
{
"value": "20240103"
}
],
"metricValues": [
{
"value": "233"
},
{
"value": "283"
},
{
"value": "79"
},
{
"value": "245"
},
{
"value": "10628"
}
]
},
{
"dimensionValues": [
{
"value": "20240102"
}
],
"metricValues": [
{
"value": "203"
},
{
"value": "269"
},
{
"value": "60"
},
{
"value": "210"
},
{
"value": "11168"
}
]
},
{
"dimensionValues": [
{
"value": "20240101"
}
],
"metricValues": [
{
"value": "191"
},
{
"value": "224"
},
{
"value": "71"
},
{
"value": "197"
},
{
"value": "8080"
}
]
}
]
}
]
}

@ -1,38 +0,0 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240222"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20200421"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20120118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -1,38 +0,0 @@
{
"reports": [
{
"columnHeader": {
"dimensions": [
"ga:date"
],
"metricHeader": {
"metricHeaderEntries": [
{
"name": "ga:pageviews",
"type": "INTEGER"
}
]
}
},
"data": {
"isDataGolden": true,
"rowCount": 849,
"rows": [
{
"dimensions": [
"20170118"
],
"metrics": [
{
"values": [
"37"
]
}
]
}
]
},
"nextPageToken": "1"
}
]
}

@ -0,0 +1,35 @@
# Account locking
This document explains the account locking feature from a technical perspective. Account locking happens when users have outgrown their accounts, and despite e-mail and dashboard alerts, don't upgrade after a grace period.
## Usage alert and grace period
The `Plausible.Workers.CheckUsage` daily background job alerts users they have reached their subscription limits. This runs for outgrown users one day after their last billing date.
When users reach the number of sites limit, or use >110% of their pageview limit for 2 consecutive billing cycles, the background job sends them an e-mail alert. The e-mail suggests a suitable subscription plan based on usage. For enterprise users, only an internal e-mail is sent to `enterprise@plausible.io`.
The user is given 7 days to upgrade their account after the alert, and this is called grace period. The background starts this grace period by adding a `users.grace_period` JSON to the user record:
```json
// SELECT grace_period FROM users LIMIT 1
{
"id": "1aa855bd-022d-4dfc-b572-6853442c3f19",
"is_over": true,
"end_date": "2022-03-09",
"allowance_required": 100,
"manual_lock": false
}
```
During this period, the following alert pops up on the dashboard for both standard and enterprise users:
![](https://files.gitbook.com/v0/b/gitbook-x-prod.appspot.com/o/spaces%2FN4GLWMwCrTuTcf31kYE9%2Fuploads%2FmsLk4CdSHKzU8TbfvaPq%2FPasted%20image%2020220909120933.png?alt=media&token=76f247a1-28cf-4d88-a0fa-06547268aee9)
If the user upgrades to a suitable plan, the grace period is removed (check `Plausible.Billing.subscription_updated/1`), otherwise [Account locking](#Account\ locking) follows. For enterprise users, the grace period has a manual lock flag set to true, meaning enterprise accounts can only be locked/unlocked through the internal CRM.
## Account locking
The grace period is checked daily by the `Plausible.Workers.LockSites` background job.
For users that expired their grace period, `sites.locked` is is set to `true`, restricting access to dashboards. This does not stop event ingestion, so users can have their stats up to date when they finally upgrade.
![](https://files.gitbook.com/v0/b/gitbook-x-prod.appspot.com/o/spaces%2FN4GLWMwCrTuTcf31kYE9%2Fuploads%2FAplurtG7UsGXMskZOlUO%2FPasted%20image%2020220909122622.png?alt=media&token=5c8156d7-d4a7-4c99-8bac-2f1e9b7d4cae)

@ -24,7 +24,6 @@ defmodule Mix.Tasks.SendPageview do
page: :string,
referrer: :string,
host: :string,
hostname: :string,
event: :string,
props: :string,
revenue_currency: :string,
@ -87,7 +86,6 @@ defmodule Mix.Tasks.SendPageview do
referrer = Keyword.get(opts, :referrer, @default_referrer)
event = Keyword.get(opts, :event, @default_event)
props = Keyword.get(opts, :props, @default_props)
hostname = Keyword.get(opts, :hostname, domain)
revenue =
if Keyword.get(opts, :revenue_currency) do
@ -99,7 +97,7 @@ defmodule Mix.Tasks.SendPageview do
%{
name: event,
url: "http://#{hostname}#{page}",
url: "http://#{domain}#{page}",
domain: domain,
referrer: referrer,
props: props,

@ -1,20 +1,5 @@
defmodule ObanErrorReporter do
require Logger
def handle_event(name, measurements, metadata, _) do
# handling telemetry event in a try/catch block
# to avoid handler detachment in the case of an error
# see https://hexdocs.pm/telemetry/telemetry.html#attach/4
try do
handle_event(name, measurements, metadata)
catch
kind, reason ->
message = Exception.format(kind, reason, __STACKTRACE__)
Logger.error(message)
end
end
defp handle_event([:oban, :job, :exception], measure, %{job: job} = meta) do
def handle_event([:oban, :job, :exception], measure, %{job: job} = meta, _) do
extra =
job
|> Map.take([:id, :args, :meta, :queue, :worker])
@ -25,43 +10,40 @@ defmodule ObanErrorReporter do
Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
end
defp handle_event([:oban, :notifier, :exception], _timing, meta) do
def handle_event([:oban, :notifier, :exception], _timing, meta, _) do
extra = Map.take(meta, ~w(channel payload)a)
Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
end
defp handle_event([:oban, :plugin, :exception], _timing, meta) do
def handle_event([:oban, :plugin, :exception], _timing, meta, _) do
extra = Map.take(meta, ~w(plugin)a)
Sentry.capture_exception(meta.reason, stacktrace: meta.stacktrace, extra: extra)
end
defp on_job_exception(%Oban.Job{
queue: "analytics_imports",
args: %{"import_id" => import_id},
queue: "google_analytics_imports",
args: %{"site_id" => site_id},
state: "executing",
attempt: attempt,
max_attempts: max_attempts
})
when attempt >= max_attempts do
site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
site = Plausible.Repo.get(Plausible.Site, site_id)
if site_import do
Plausible.Workers.ImportAnalytics.import_fail(site_import)
if site do
Plausible.Workers.ImportGoogleAnalytics.import_failed(site)
end
end
defp on_job_exception(%Oban.Job{
queue: "analytics_imports",
args: %{"import_id" => import_id},
queue: "google_analytics_imports",
args: %{"site_id" => site_id},
state: "executing"
}) do
site_import = Plausible.Repo.get(Plausible.Imported.SiteImport, import_id)
if site_import do
Plausible.Workers.ImportAnalytics.import_fail_transient(site_import)
end
site = Plausible.Repo.get(Plausible.Site, site_id)
Plausible.Purge.delete_imported_stats!(site)
end
defp on_job_exception(_job), do: :ignore

@ -2,15 +2,11 @@ defmodule Plausible.Application do
@moduledoc false
use Application
use Plausible
require Logger
def start(_type, _args) do
on_full_build(do: Plausible.License.ensure_valid_license())
children = [
Plausible.Cache.Stats,
Plausible.Repo,
Plausible.ClickhouseRepo,
Plausible.IngestRepo,
@ -24,74 +20,15 @@ defmodule Plausible.Application do
Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer),
ReferrerBlocklist,
Plausible.Cache.Adapter.child_spec(:user_agents, :cache_user_agents,
ttl_check_interval: :timer.seconds(5),
global_ttl: :timer.minutes(60)
Supervisor.child_spec({Cachex, name: :user_agents, limit: 10_000, stats: true},
id: :cachex_user_agents
),
Plausible.Cache.Adapter.child_spec(:sessions, :cache_sessions,
ttl_check_interval: :timer.seconds(1),
global_ttl: :timer.minutes(30)
Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},
id: :cachex_sessions
),
{Plausible.Site.Cache, ttl_check_interval: false},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Site.Cache.All,
cache_impl: Plausible.Site.Cache,
interval: :timer.minutes(15) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Site.Cache.RecentlyUpdated,
cache_impl: Plausible.Site.Cache,
interval: :timer.seconds(30),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Shield.IPRuleCache, ttl_check_interval: false},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.IPRuleCache.All,
cache_impl: Plausible.Shield.IPRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.IPRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.IPRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Shield.CountryRuleCache, ttl_check_interval: false},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.CountryRuleCache.All,
cache_impl: Plausible.Shield.CountryRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.CountryRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.CountryRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Shield.PageRuleCache, ttl_check_interval: false, ets_options: [:bag]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.All,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.minutes(3) + Enum.random(1..:timer.seconds(10)),
warmer_fn: :refresh_all
]},
{Plausible.Cache.Warmer,
[
child_name: Plausible.Shield.PageRuleCache.RecentlyUpdated,
cache_impl: Plausible.Shield.PageRuleCache,
interval: :timer.seconds(35),
warmer_fn: :refresh_updated_recently
]},
{Plausible.Site.Cache, []},
{Plausible.Site.Cache.Warmer.All, []},
{Plausible.Site.Cache.Warmer.RecentlyUpdated, []},
{Plausible.Auth.TOTP.Vault, key: totp_vault_key()},
PlausibleWeb.Endpoint,
{Oban, Application.get_env(:plausible, Oban)},
@ -137,10 +74,10 @@ defmodule Plausible.Application do
defp maybe_add_sentry_pool(pool_config) do
case Sentry.Config.dsn() do
{"http" <> _rest = url, _, _} ->
Map.put(pool_config, url, size: 50)
dsn when is_binary(dsn) ->
Map.put(pool_config, dsn, size: 50)
nil ->
_ ->
pool_config
end
end
@ -186,6 +123,16 @@ defmodule Plausible.Application do
)
end
def report_cache_stats() do
case Cachex.stats(:user_agents) do
{:ok, stats} ->
Logger.info("User agent cache stats: #{inspect(stats)}")
e ->
IO.puts("Unable to show cache stats: #{inspect(e)}")
end
end
defp setup_opentelemetry() do
OpentelemetryPhoenix.setup()
OpentelemetryEcto.setup([:plausible, :repo])

@ -8,7 +8,7 @@ defmodule Plausible.Auth.GracePeriod do
background with `Plausible.Workers.LockSites`.
The grace period can also be manual, without an end date, being controlled
manually from the CRM, and not by the background site locker job. This is
manually from the CRM, and not by the background site locker job. This is
useful for enterprise subscriptions.
"""
@ -17,23 +17,26 @@ defmodule Plausible.Auth.GracePeriod do
@type t() :: %__MODULE__{
end_date: Date.t() | nil,
allowance_required: non_neg_integer(),
is_over: boolean(),
manual_lock: boolean()
}
embedded_schema do
field :end_date, :date
field :allowance_required, :integer
field :is_over, :boolean
field :manual_lock, :boolean
end
@spec start_changeset(User.t()) :: Ecto.Changeset.t()
@spec start_changeset(User.t(), non_neg_integer()) :: Ecto.Changeset.t()
@doc """
Starts a account locking grace period of 7 days by changing the User struct.
"""
def start_changeset(%User{} = user) do
def start_changeset(%User{} = user, allowance_required) do
grace_period = %__MODULE__{
end_date: Timex.shift(Timex.today(), days: 7),
allowance_required: allowance_required,
is_over: false,
manual_lock: false
}
@ -41,15 +44,16 @@ defmodule Plausible.Auth.GracePeriod do
Ecto.Changeset.change(user, grace_period: grace_period)
end
@spec start_manual_lock_changeset(User.t()) :: Ecto.Changeset.t()
@spec start_manual_lock_changeset(User.t(), non_neg_integer()) :: Ecto.Changeset.t()
@doc """
Starts a manual account locking grace period by changing the User struct.
Starts a manual account locking grace period by changing the User struct.
Manual locking means the grace period can only be removed manually from the
CRM.
"""
def start_manual_lock_changeset(%User{} = user) do
def start_manual_lock_changeset(%User{} = user, allowance_required) do
grace_period = %__MODULE__{
end_date: nil,
allowance_required: allowance_required,
is_over: false,
manual_lock: true
}

@ -118,7 +118,7 @@ defmodule Plausible.Auth.UserAdmin do
user.subscription ->
PlausibleWeb.AuthView.present_subscription_status(user.subscription.status)
Plausible.Users.on_trial?(user) ->
Plausible.Billing.on_trial?(user) ->
"On trial"
true ->

@ -2,7 +2,6 @@ defmodule Plausible.Billing do
use Plausible
use Plausible.Repo
require Plausible.Billing.Subscription.Status
alias Plausible.Billing.Subscriptions
alias Plausible.Billing.{Subscription, Plans, Quota}
alias Plausible.Auth.User
@ -95,26 +94,40 @@ defmodule Plausible.Billing do
| :no_upgrade_needed
def check_needs_to_upgrade(user) do
user = Plausible.Users.with_subscription(user)
trial_is_over = user.trial_expiry_date && Timex.before?(user.trial_expiry_date, Timex.today())
subscription_active = subscription_is_active?(user.subscription)
trial_over? =
not is_nil(user.trial_expiry_date) and
Date.before?(user.trial_expiry_date, Date.utc_today())
cond do
!user.trial_expiry_date && !subscription_active -> {:needs_to_upgrade, :no_trial}
trial_is_over && !subscription_active -> {:needs_to_upgrade, :no_active_subscription}
Plausible.Auth.GracePeriod.expired?(user) -> {:needs_to_upgrade, :grace_period_ended}
true -> :no_upgrade_needed
end
end
subscription_active? = Subscriptions.active?(user.subscription)
def subscription_is_active?(%Subscription{status: Subscription.Status.active()}), do: true
def subscription_is_active?(%Subscription{status: Subscription.Status.past_due()}), do: true
cond do
is_nil(user.trial_expiry_date) and not subscription_active? ->
{:needs_to_upgrade, :no_trial}
def subscription_is_active?(%Subscription{status: Subscription.Status.deleted()} = subscription) do
subscription.next_bill_date && !Timex.before?(subscription.next_bill_date, Timex.today())
end
trial_over? and not subscription_active? ->
{:needs_to_upgrade, :no_active_subscription}
def subscription_is_active?(%Subscription{}), do: false
def subscription_is_active?(nil), do: false
Plausible.Auth.GracePeriod.expired?(user) ->
{:needs_to_upgrade, :grace_period_ended}
on_full_build do
def on_trial?(%User{trial_expiry_date: nil}), do: false
true ->
:no_upgrade_needed
def on_trial?(user) do
user = Plausible.Users.with_subscription(user)
!subscription_is_active?(user.subscription) && trial_days_left(user) >= 0
end
else
def on_trial?(_), do: false
end
def trial_days_left(user) do
Timex.diff(user.trial_expiry_date, Timex.today(), :days)
end
defp handle_subscription_created(params) do
@ -177,19 +190,13 @@ defmodule Plausible.Billing do
amount =
:erlang.float_to_binary(api_subscription["next_payment"]["amount"] / 1, decimals: 2)
subscription =
subscription
|> Subscription.changeset(%{
next_bill_amount: amount,
next_bill_date: api_subscription["next_payment"]["date"],
last_bill_date: api_subscription["last_payment"]["date"]
})
|> Repo.update!()
|> Repo.preload(:user)
Plausible.Users.update_accept_traffic_until(subscription.user)
subscription
|> Subscription.changeset(%{
next_bill_amount: amount,
next_bill_date: api_subscription["next_payment"]["date"],
last_bill_date: api_subscription["last_payment"]["date"]
})
|> Repo.update!()
end
end
@ -221,10 +228,25 @@ defmodule Plausible.Billing do
defp present?(nil), do: false
defp present?(_), do: true
defp remove_grace_period(%User{} = user) do
user
|> Plausible.Auth.GracePeriod.remove_changeset()
|> Repo.update!()
defp maybe_remove_grace_period(%User{} = user) do
alias Plausible.Auth.GracePeriod
case user.grace_period do
%GracePeriod{allowance_required: allowance_required} ->
new_monthly_pageview_limit =
Plausible.Billing.Quota.monthly_pageview_limit(user)
if new_monthly_pageview_limit > allowance_required do
user
|> Plausible.Auth.GracePeriod.remove_changeset()
|> Repo.update!()
else
user
end
_ ->
user
end
end
@spec format_price(Money.t()) :: String.t()
@ -254,7 +276,7 @@ defmodule Plausible.Billing do
user
|> Plausible.Users.update_accept_traffic_until()
|> remove_grace_period()
|> maybe_remove_grace_period()
|> Plausible.Users.maybe_reset_next_upgrade_override()
|> tap(&Plausible.Billing.SiteLocker.update_sites_for/1)
|> maybe_adjust_api_key_limits()

@ -87,18 +87,6 @@ defmodule Plausible.Billing.Feature do
@features
end
@doc """
Lists all the feature short names, e.g. RevenueGoals
"""
defmacro list_short_names() do
@features
|> Enum.map(fn mod ->
Module.split(mod)
|> List.last()
|> String.to_atom()
end)
end
@doc false
defmacro __using__(opts \\ []) do
quote location: :keep do
@ -206,7 +194,7 @@ defmodule Plausible.Billing.Feature.StatsAPI do
@doc """
Checks whether the user has access to Stats API or not.
Before the business tier, users who had not yet started their trial had
Before the the business tier, users who had not yet started their trial had
access to Stats API. With the business tier work, access is blocked and they
must either start their trial or subscribe to a plan. This is common when a
site owner invites a new user. In such cases, using the owner's API key is

@ -5,10 +5,9 @@ defmodule Plausible.Billing.Quota do
use Plausible
import Ecto.Query
alias Plausible.Users
alias Plausible.Auth.User
alias Plausible.Site
alias Plausible.Billing.{Plan, Plans, Subscription, Subscriptions, EnterprisePlan, Feature}
alias Plausible.Billing.{Plan, Plans, Subscription, EnterprisePlan, Feature}
alias Plausible.Billing.Feature.{Goals, RevenueGoals, Funnels, Props, StatsAPI}
@type limit() :: :site_limit | :pageview_limit | :team_member_limit
@ -26,10 +25,6 @@ defmodule Plausible.Billing.Quota do
total: non_neg_integer()
}
@pageview_allowance_margin 0.1
def pageview_allowance_margin(), do: @pageview_allowance_margin
def usage(user, opts \\ []) do
basic_usage = %{
monthly_pageviews: monthly_pageview_usage(user),
@ -60,7 +55,7 @@ defmodule Plausible.Billing.Quota do
end
defp get_site_limit_from_plan(user) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
case Plans.get_subscription_plan(user.subscription) do
%{site_limit: site_limit} -> site_limit
@ -71,7 +66,7 @@ defmodule Plausible.Billing.Quota do
@spec team_member_limit(User.t()) :: non_neg_integer()
def team_member_limit(user) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
case Plans.get_subscription_plan(user.subscription) do
%{team_member_limit: limit} -> limit
@ -103,7 +98,7 @@ defmodule Plausible.Billing.Quota do
in a background job instead (see `check_usage.ex`).
"""
def ensure_can_add_new_site(user) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
case Plans.get_subscription_plan(user.subscription) do
%EnterprisePlan{} ->
@ -123,7 +118,7 @@ defmodule Plausible.Billing.Quota do
@spec monthly_pageview_limit(User.t() | Subscription.t()) ::
non_neg_integer() | :unlimited
def monthly_pageview_limit(%User{} = user) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
monthly_pageview_limit(user.subscription)
end
@ -181,7 +176,7 @@ defmodule Plausible.Billing.Quota do
end
def monthly_pageview_usage(user, site_ids) do
active_subscription? = Subscriptions.active?(user.subscription)
active_subscription? = Plausible.Billing.subscription_is_active?(user.subscription)
if active_subscription? && user.subscription.last_bill_date do
[:current_cycle, :last_cycle, :penultimate_cycle]
@ -218,7 +213,7 @@ defmodule Plausible.Billing.Quota do
end
def usage_cycle(user, cycle, owned_site_ids, today) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
last_bill_date = user.subscription.last_bill_date
normalized_last_bill_date =
@ -266,28 +261,15 @@ defmodule Plausible.Billing.Quota do
* Users are counted uniquely - i.e. even if an account is associated with
many sites owned by the given user, they still count as one team member.
* Specific e-mails can be excluded from the count, so that where necessary,
we can ensure inviting the same person(s) to more than 1 sites is allowed
"""
def team_member_usage(user, opts \\ []) do
{:ok, opts} = Keyword.validate(opts, site: nil, exclude_emails: [])
user
|> team_member_usage_query(opts)
|> Plausible.Repo.aggregate(:count)
def team_member_usage(user) do
Plausible.Repo.aggregate(team_member_usage_query(user), :count)
end
defp team_member_usage_query(user, opts) do
@doc false
def team_member_usage_query(user, site \\ nil) do
owned_sites_query = owned_sites_query(user)
excluded_emails =
opts
|> Keyword.get(:exclude_emails, [])
|> List.wrap()
site = opts[:site]
owned_sites_query =
if site do
where(owned_sites_query, [os], os.site_id == ^site.id)
@ -303,27 +285,12 @@ defmodule Plausible.Billing.Quota do
where: sm.role != :owner,
select: u.email
team_members_query =
if excluded_emails != [] do
team_members_query |> where([..., u], u.email not in ^excluded_emails)
else
team_members_query
end
query =
from i in Plausible.Auth.Invitation,
inner_join: os in subquery(owned_sites_query),
on: i.site_id == os.site_id,
where: i.role != :owner,
select: i.email,
union: ^team_members_query
if excluded_emails != [] do
query
|> where([i], i.email not in ^excluded_emails)
else
query
end
from i in Plausible.Auth.Invitation,
inner_join: os in subquery(owned_sites_query),
on: i.site_id == os.site_id,
where: i.role != :owner,
select: i.email,
union: ^team_members_query
end
@spec features_usage(User.t() | Site.t()) :: [atom()]
@ -443,7 +410,7 @@ defmodule Plausible.Billing.Quota do
else
case usage do
%{last_30_days: %{total: total}} ->
!within_limit?(total, pageview_limit_with_margin(plan, opts))
!within_limit?(total, pageview_limit_with_margin(plan))
billing_cycles_usage ->
Plausible.Workers.CheckUsage.exceeds_last_two_usage_cycles?(
@ -454,9 +421,9 @@ defmodule Plausible.Billing.Quota do
end
end
defp pageview_limit_with_margin(%{monthly_pageview_limit: limit}, opts) do
margin = Keyword.get(opts, :pageview_allowance_margin, @pageview_allowance_margin)
ceil(limit * (1 + margin))
defp pageview_limit_with_margin(%{monthly_pageview_limit: limit}) do
allowance_margin = if limit == 10_000, do: 0.3, else: 0.15
ceil(limit * (1 + allowance_margin))
end
@doc """
@ -464,24 +431,13 @@ defmodule Plausible.Billing.Quota do
ability to use all features during their trial.
"""
def allowed_features_for(user) do
user = Users.with_subscription(user)
user = Plausible.Users.with_subscription(user)
case Plans.get_subscription_plan(user.subscription) do
%EnterprisePlan{features: features} ->
features
%Plan{features: features} ->
features
:free_10k ->
[Goals, Props, StatsAPI]
nil ->
if Users.on_trial?(user) do
Feature.list()
else
[Goals]
end
%EnterprisePlan{features: features} -> features
%Plan{features: features} -> features
:free_10k -> [Goals, Props, StatsAPI]
nil -> Feature.list()
end
end

@ -4,17 +4,6 @@ defmodule Plausible.Billing.Subscriptions do
require Plausible.Billing.Subscription.Status
alias Plausible.Billing.Subscription
def active?(%Subscription{status: Subscription.Status.active()}), do: true
def active?(%Subscription{status: Subscription.Status.past_due()}), do: true
def active?(%Subscription{status: Subscription.Status.deleted()} = subscription) do
not is_nil(subscription.next_bill_date) and
not Date.before?(subscription.next_bill_date, Date.utc_today())
end
def active?(%Subscription{}), do: false
def active?(nil), do: false
@spec expired?(Subscription.t()) :: boolean()
@doc """
Returns whether the given subscription is expired. That means that the

@ -1,191 +0,0 @@
defmodule Plausible.Cache do
@moduledoc """
Caching interface specific for Plausible. Usage:
use Plausible.Cache
# - Implement the callbacks required
# - Optionally override `unwrap_cache_keys/1`
# - Populate the cache with `Plausible.Cache.Warmer`
Serves as a wrapper around `Plausible.Cache.Adapter`, where the underlying
implementation can be transparently swapped.
Even though normally the relevant Adapter processes are started, cache access is disabled
during tests via the `:plausible, #{__MODULE__}, enabled: bool()` application env key.
This can be overridden on case by case basis, using the child specs options.
The `base_db_query/0` callback is used to generate the base query that is
executed on every cache refresh.
There are two modes of refresh operation: `:all` and `:updated_recently`;
the former will invoke the query as is and clear all the existing entries,
while the latter will attempt to limit the query to only the records that
have been updated in the last 15 minutes and try to merge the new results with
existing cache entries.
Both refresh modes are normally executed periodically from within a warmer process;
see: `Plausible.Cache.Warmer`. The reason for two modes is that the latter is lighter
on the database and can be executed more frequently.
When Cache is disabled via application env, the `get/1` function
falls back to pure database lookups (implemented via `get_from_source/1` callback.
This should help with introducing cached lookups in existing code,
so that no existing tests should break.
Refreshing the cache emits telemetry event defined as per `telemetry_event_refresh/2`.
"""
@doc "Unique cache name, used by underlying implementation"
@callback name() :: atom()
@doc "Supervisor child id, must be unique within the supervision tree"
@callback child_id() :: atom()
@doc "Counts all items at the source, an aggregate query most likely"
@callback count_all() :: integer()
@doc "Optionally unwraps the keys of the cache items, in case one item is stored under multiple keys"
@callback unwrap_cache_keys([any()]) :: [{any(), any()}]
@doc "Returns the base Ecto query used to refresh the cache"
@callback base_db_query() :: Ecto.Query.t()
@doc "Retrieves the item from the source, in case the cache is disabled"
@callback get_from_source(any()) :: any()
@doc "Looks for application env value at `:plausible, #{__MODULE__}, enabled: bool()`"
def enabled?() do
Application.fetch_env!(:plausible, __MODULE__)[:enabled] == true
end
# credo:disable-for-this-file Credo.Check.Refactor.LongQuoteBlocks
defmacro __using__(_opts) do
quote do
require Logger
@behaviour Plausible.Cache
@modes [:all, :updated_recently]
alias Plausible.Cache.Adapter
@spec get(any(), Keyword.t()) :: any() | nil
def get(key, opts \\ []) when is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name())
force? = Keyword.get(opts, :force?, false)
if Plausible.Cache.enabled?() or force? do
Adapter.get(cache_name, key)
else
get_from_source(key)
end
end
@spec get_or_store(any(), (-> any()), Keyword.t()) :: any() | nil
def get_or_store(key, fallback_fn, opts \\ [])
when is_function(fallback_fn, 0) and is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name())
force? = Keyword.get(opts, :force?, false)
if Plausible.Cache.enabled?() or force? do
Adapter.get(cache_name, key, fallback_fn)
else
get_from_source(key) || fallback_fn.()
end
end
def unwrap_cache_keys(items), do: items
defoverridable unwrap_cache_keys: 1
@spec refresh_all(Keyword.t()) :: :ok
def refresh_all(opts \\ []) do
refresh(
:all,
base_db_query(),
Keyword.put(opts, :delete_stale_items?, true)
)
end
@spec refresh_updated_recently(Keyword.t()) :: :ok
def refresh_updated_recently(opts \\ []) do
recently_updated_query =
from [s, _rg] in base_db_query(),
order_by: [asc: s.updated_at],
where: s.updated_at > ago(^15, "minute")
refresh(
:updated_recently,
recently_updated_query,
Keyword.put(opts, :delete_stale_items?, false)
)
end
@spec merge_items(new_items :: [any()], opts :: Keyword.t()) :: :ok
def merge_items(new_items, opts \\ [])
def merge_items([], _), do: :ok
def merge_items(new_items, opts) do
new_items = unwrap_cache_keys(new_items)
cache_name = Keyword.get(opts, :cache_name, name())
:ok = Adapter.put_many(cache_name, new_items)
if Keyword.get(opts, :delete_stale_items?, true) do
old_keys = Adapter.keys(cache_name)
new = MapSet.new(Enum.into(new_items, [], fn {k, _} -> k end))
old = MapSet.new(old_keys)
old
|> MapSet.difference(new)
|> Enum.each(fn k ->
Adapter.delete(cache_name, k)
end)
end
:ok
end
@spec child_spec(Keyword.t()) :: Supervisor.child_spec()
def child_spec(opts) do
cache_name = Keyword.get(opts, :cache_name, name())
child_id = Keyword.get(opts, :child_id, child_id())
Adapter.child_spec(cache_name, child_id, opts)
end
@doc """
Ensures the cache has non-zero size unless no items exist.
Useful for orchestrating app startup to prevent the service
going up asynchronously with an empty cache.
"""
@spec ready?(atom()) :: boolean
def ready?(cache_name \\ name()) do
case size(cache_name) do
n when is_integer(n) and n > 0 ->
true
0 ->
count_all() == 0
_ ->
false
end
end
defdelegate size(cache_name \\ name()), to: Plausible.Cache.Adapter
defdelegate hit_rate(cache_name \\ name()), to: Plausible.Cache.Stats
@spec telemetry_event_refresh(atom(), atom()) :: list(atom())
def telemetry_event_refresh(cache_name \\ name(), mode) when mode in @modes do
[:plausible, :cache, cache_name, :refresh, mode]
end
defp refresh(mode, query, opts) when mode in @modes do
cache_name = Keyword.get(opts, :cache_name, name())
Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(
telemetry_event_refresh(cache_name, mode),
fn ->
items = Plausible.Repo.all(query)
:ok = merge_items(items, opts)
end
)
:ok
end
end
end
end

@ -1,136 +0,0 @@
defmodule Plausible.Cache.Adapter do
@moduledoc """
Interface for the underlying cache implementation.
Currently: ConCache
Using the Adapter module directly, the user must ensure that the relevant
processes are available to use, which is normally done via the child specification.
"""
require Logger
@spec child_spec(atom(), atom(), Keyword.t()) :: Supervisor.child_spec()
def child_spec(name, child_id, opts \\ [])
when is_atom(name) and is_atom(child_id) and is_list(opts) do
cache_name = Keyword.get(opts, :cache_name, name)
child_id = Keyword.get(opts, :child_id, child_id)
ttl_check_interval = Keyword.get(opts, :ttl_check_interval, false)
opts =
opts
|> Keyword.put(:name, cache_name)
|> Keyword.put(:ttl_check_interval, ttl_check_interval)
Supervisor.child_spec(
{ConCache, opts},
id: child_id
)
end
@spec size(atom()) :: non_neg_integer() | nil
def size(cache_name) do
ConCache.size(cache_name)
catch
:exit, _ -> nil
end
@spec get(atom(), any()) :: any()
def get(cache_name, key) do
result = ConCache.get(cache_name, key)
if is_nil(result), do: miss(cache_name), else: hit(cache_name)
result
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
nil
end
@spec get(atom(), any(), (-> any())) :: any()
def get(cache_name, key, fallback_fn) do
cache = ConCache.Owner.cache(cache_name)
case ConCache.Operations.get(cache, key) do
nil ->
get_or_store_isolated(cache, cache_name, key, fallback_fn)
value ->
hit(cache_name)
value
end
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
nil
end
@spec put(atom(), any(), any()) :: any()
def put(cache_name, key, value) do
:ok = ConCache.put(cache_name, key, value)
value
catch
:exit, _ ->
Logger.error("Error putting a key to '#{cache_name}'")
nil
end
@spec put_many(atom(), [any()]) :: :ok
def put_many(cache_name, items) when is_list(items) do
true = :ets.insert(ConCache.ets(cache_name), items)
:ok
catch
:exit, _ ->
Logger.error("Error putting keys to '#{cache_name}'")
:ok
end
@spec delete(atom(), any()) :: :ok
def delete(cache_name, key) do
ConCache.dirty_delete(cache_name, key)
catch
:exit, _ ->
Logger.error("Error deleting a key in '#{cache_name}'")
:ok
end
@spec keys(atom()) :: Enumerable.t()
def keys(cache_name) do
ets = ConCache.ets(cache_name)
Stream.resource(
fn -> :ets.first(ets) end,
fn
:"$end_of_table" -> {:halt, nil}
prev_key -> {[prev_key], :ets.next(ets, prev_key)}
end,
fn _ -> :ok end
)
catch
:exit, _ ->
Logger.error("Error retrieving key from '#{inspect(cache_name)}'")
[]
end
defp hit(cache_name) do
Plausible.Cache.Stats.record_hit(cache_name)
end
defp miss(cache_name) do
Plausible.Cache.Stats.record_miss(cache_name)
end
defp get_or_store_isolated(cache, cache_name, key, fun) do
ConCache.isolated(cache_name, key, fn ->
case ConCache.Operations.get(cache, key) do
nil ->
new_value = fun.()
ConCache.Operations.dirty_put(cache, key, new_value)
miss(cache_name)
new_value
existing ->
hit(cache_name)
existing
end
end)
end
end

@ -1,84 +0,0 @@
defmodule Plausible.Cache.Stats do
@moduledoc """
Keeps track of hit/miss ratio for various caches.
"""
use GenServer
@hit :hit
@miss :miss
@telemetry_hit [:plausible, :cache, :adapter, @hit]
@telemetry_miss [:plausible, :cache, :adapter, @miss]
@telemetry_events [@telemetry_hit, @telemetry_miss]
def start_link(_opts) do
GenServer.start_link(__MODULE__, nil)
end
def record_hit(cache_name) do
:telemetry.execute(@telemetry_hit, %{}, %{cache_name: cache_name})
end
def record_miss(cache_name) do
:telemetry.execute(@telemetry_miss, %{}, %{cache_name: cache_name})
end
def init(nil) do
__MODULE__ =
:ets.new(__MODULE__, [
:public,
:named_table,
:set,
read_concurrency: true,
write_concurrency: true
])
:telemetry.attach_many(
"plausible-cache-stats",
@telemetry_events,
&__MODULE__.handle_telemetry_event/4,
nil
)
{:ok, nil}
end
def handle_telemetry_event(@telemetry_hit, _measurments, %{cache_name: cache_name}, _) do
bump(cache_name, @hit)
end
def handle_telemetry_event(@telemetry_miss, _measurments, %{cache_name: cache_name}, _) do
bump(cache_name, @miss)
end
def gather(cache_name) do
{:ok,
%{
hit_rate: hit_rate(cache_name),
count: size(cache_name) || 0
}}
end
defdelegate size(cache_name), to: Plausible.Cache.Adapter
def bump(cache_name, type) do
:ets.update_counter(
__MODULE__,
{cache_name, type},
1,
{{cache_name, type}, 0}
)
end
def hit_rate(cache_name) do
hit = :ets.lookup_element(__MODULE__, {cache_name, @hit}, 2, 0)
miss = :ets.lookup_element(__MODULE__, {cache_name, @miss}, 2, 0)
hit_miss = hit + miss
if hit_miss == 0 do
0.0
else
hit / hit_miss * 100
end
end
end

@ -1,83 +0,0 @@
defmodule Plausible.Cache.Warmer do
@moduledoc """
A periodic cache warmer.
Child specification options available:
* `cache_impl` - module expected to implement `Plausible.Cache` behaviour
* `interval` - the number of milliseconds for each warm-up cycle
* `cache_name` - defaults to cache_impl.name() but can be overridden for testing
* `force_start?` - enforcess process startup for testing, even if it's barred
by `Plausible.Cache.enabled?`. This is useful for avoiding issues with DB ownership
and async tests.
* `warmer_fn` - by convention, either `:refresh_all` or `:refresh_updated_recently`,
both are automatically provided by `cache_impl` module. Technically any exported
or captured function will work, if need be.
See tests for more comprehensive examples.
"""
@behaviour :gen_cycle
require Logger
@spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore
def child_spec(opts) do
child_name = Keyword.get(opts, :child_name, __MODULE__)
%{
id: child_name,
start: {:gen_cycle, :start_link, [{:local, child_name}, __MODULE__, opts]}
}
end
@impl true
def init_cycle(opts) do
cache_impl = Keyword.fetch!(opts, :cache_impl)
cache_name = Keyword.get(opts, :cache_name, cache_impl.name())
interval = Keyword.fetch!(opts, :interval)
warmer_fn =
case Keyword.fetch!(opts, :warmer_fn) do
f when is_function(f, 1) ->
f
f when is_atom(f) ->
true = function_exported?(cache_impl, f, 1)
Function.capture(cache_impl, f, 1)
end
force_start? = Keyword.get(opts, :force_start?, false)
if Plausible.Cache.enabled?() or force_start? do
Logger.info(
"#{__MODULE__} initializing #{inspect(warmer_fn)} #{cache_name} with interval #{interval}..."
)
{:ok,
{interval,
opts
|> Keyword.put(:cache_name, cache_name)
|> Keyword.put(:warmer_fn, warmer_fn)}}
else
:ignore
end
end
@impl true
def handle_cycle(opts) do
cache_name = Keyword.fetch!(opts, :cache_name)
warmer_fn = Keyword.fetch!(opts, :warmer_fn)
Logger.info("#{__MODULE__} running #{inspect(warmer_fn)} on #{cache_name}...")
warmer_fn.(opts)
{:continue_hibernated, opts}
end
@impl true
def handle_info(_msg, state) do
{:continue, state}
end
end

@ -15,15 +15,6 @@ defmodule Plausible.ClickhouseEventV2 do
field :session_id, Ch, type: "UInt64"
field :timestamp, :naive_datetime
field :"meta.key", {:array, :string}
field :"meta.value", {:array, :string}
field :revenue_source_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_source_currency, Ch, type: "FixedString(3)"
field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_reporting_currency, Ch, type: "FixedString(3)"
# Session attributes
field :referrer, :string
field :referrer_source, :string
field :utm_medium, :string
@ -42,6 +33,16 @@ defmodule Plausible.ClickhouseEventV2 do
field :operating_system_version, Ch, type: "LowCardinality(String)"
field :browser, Ch, type: "LowCardinality(String)"
field :browser_version, Ch, type: "LowCardinality(String)"
field :"meta.key", {:array, :string}
field :"meta.value", {:array, :string}
field :revenue_source_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_source_currency, Ch, type: "FixedString(3)"
field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))"
field :revenue_reporting_currency, Ch, type: "FixedString(3)"
field :transferred_from, :string
end
def new(attrs) do
@ -55,6 +56,22 @@ defmodule Plausible.ClickhouseEventV2 do
:pathname,
:user_id,
:timestamp,
:operating_system,
:operating_system_version,
:browser,
:browser_version,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:"meta.key",
:"meta.value",
:revenue_source_amount,
@ -65,28 +82,4 @@ defmodule Plausible.ClickhouseEventV2 do
)
|> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
end
@session_properties [
:session_id,
:referrer,
:referrer_source,
:utm_medium,
:utm_source,
:utm_campaign,
:utm_content,
:utm_term,
:country_code,
:subdivision1_code,
:subdivision2_code,
:city_geoname_id,
:screen_size,
:operating_system,
:operating_system_version,
:browser,
:browser_version
]
def merge_session(%__MODULE__{} = event, session) do
Map.merge(event, Map.take(session, @session_properties))
end
end

@ -25,4 +25,34 @@ defmodule Plausible.ClickhouseRepo do
|> Enum.to_list()
|> Keyword.values()
end
@doc """
Automatically adds tags as query log_comment setting for clickhouse queries.
These tags include information like plausible.site.id and http.route.
The tags will be exposed in system.query_log table log_comment column and can be used
for query performance analysis.
The tags are indirectly fetched from OpenTelemetry traces to avoid duplicating data.
"""
@impl true
def prepare_query(_operation, query, opts) do
setting = {:log_comment, Jason.encode!(current_trace_attributes())}
opts = Keyword.update(opts, :settings, [setting], fn settings -> [setting | settings] end)
{query, opts}
end
defp current_trace_attributes() do
try do
:ets.tab2list(:otel_span_table)
|> Enum.at(0)
|> elem(9)
|> elem(4)
rescue
_ -> %{}
end
end
end

@ -43,7 +43,6 @@ defmodule Plausible.ClickhouseSessionV2 do
field :is_bounce, BoolUInt8
field :entry_page, :string
field :exit_page, :string
field :exit_page_hostname, :string
field :pageviews, Ch, type: "Int32"
field :events, Ch, type: "Int32"
field :sign, Ch, type: "Int8"

@ -62,21 +62,16 @@ defmodule Plausible.DataMigration do
|> EEx.eval_file(assigns: assigns)
end
def run_sql(name, assigns \\ [], options \\ []) do
def run_sql(name, assigns \\ []) do
query = unwrap(name, assigns)
do_run(name, query, options)
do_run(name, query)
end
defp do_run(name, query, options \\ []) do
case @repo.query(query, [], [timeout: :infinity] ++ options) do
{:ok, res} ->
IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
IO.puts(String.duplicate("-", 78))
{:ok, res}
result ->
result
end
defp do_run(name, query) do
{:ok, res} = @repo.query(query, [], timeout: :infinity)
IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
IO.puts(String.duplicate("-", 78))
{:ok, res}
end
defp unwrap_with_io(name, assigns) do

@ -1,166 +0,0 @@
defmodule Plausible.DataMigration.PopulateEventSessionColumns do
@moduledoc """
Populates event session columns with data from sessions table.
Run via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.run"
Kill via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.kill"
Monitor via ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.report_progress"
Suggested to run in a screen/tmux session to be able to easily monitor
SQL files available at: priv/data_migrations/PopulateEventSessionColumns/sql
"""
use Plausible.DataMigration, dir: "PopulateEventSessionColumns", repo: Plausible.IngestRepo
require Logger
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#cache for meaning of these defaults
@default_dictionary_config %{
lifetime: 600_000,
size_in_cells: 1_000_000,
max_threads_for_updates: 6
}
def run(opts \\ []) do
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
{:ok, _} =
run_sql("create-sessions-dictionary",
cluster?: cluster?,
dictionary_connection_params:
Keyword.get(opts, :dictionary_connection_string, dictionary_connection_params()),
dictionary_config: dictionary_config(opts)
)
{partitions, _, _, _} = get_partitions(opts)
IO.puts("Starting mutation on #{length(partitions)} partition(s)")
for partition <- partitions do
{:ok, _} =
run_sql("update-table", [cluster?: cluster?, partition: partition],
settings: [allow_nondeterministic_mutations: 1]
)
end
wait_until_mutations_complete(opts)
IO.puts("Mutations seem done, cleaning up!")
{:ok, _} = run_sql("drop-sessions-dictionary", cluster?: cluster?)
end
def kill(opts \\ []) do
cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")
report_progress(opts)
IO.puts("Killing running mutations")
{:ok, _} = run_sql("kill-running-mutations", cluster?: cluster?)
end
def wait_until_mutations_complete(opts \\ []) do
Process.sleep(5_000)
in_progress? = report_progress(opts)
if in_progress? do
wait_until_mutations_complete(opts)
end
end
def report_progress(opts \\ []) do
{partitions, parts, min_partition, max_partition} = get_partitions(opts)
{:ok, %{rows: mutation_results}} =
run_sql("get-mutations-progress",
min_partition: min_partition,
max_partition: max_partition
)
[
[
mutations,
parts_to_do,
running_for,
total_size,
todo_size,
progress,
latest_fail_reason,
_,
_
]
] =
mutation_results
{:ok, %{rows: [[merges]]}} = run_sql("get-merges-progress")
{:ok, %{rows: disks}} = run_sql("get-disks")
IO.puts("\n\n#{Timex.now() |> Timex.format!("{ISO:Extended}")}")
# List partitions that need to run
IO.puts(
"Progress report for partitions #{Enum.min(partitions)}-#{Enum.max(partitions)} (parts: #{length(parts)})"
)
IO.puts("Disks overview:")
for [name, path, full_space, total_space, full_percentage] <- disks do
IO.puts(
" #{name} at #{path} is at #{full_space}/#{total_space} (#{full_percentage}% full)"
)
end
IO.puts("Currently #{mutations} mutation(s) are running.")
if mutations > 0 do
IO.puts(" To do #{parts_to_do} parts, #{todo_size}")
IO.puts(" Out of #{length(parts)} parts, #{total_size}")
IO.puts(" Running for #{format_duration(running_for)}")
if progress > 0 do
estimated_time_left = round(running_for / progress / 100 - running_for)
IO.puts(" Estimated #{progress}% done, #{format_duration(estimated_time_left)} left")
end
if latest_fail_reason do
IO.puts(" Some mutations might be failing. ClickHouse report: #{latest_fail_reason}")
end
end
IO.puts("Currently #{merges} merge(s) are running relating to mutations.")
mutations > 0
end
defp dictionary_config(opts) do
@default_dictionary_config
|> Map.merge(Keyword.get(opts, :dictionary_config, %{}))
end
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context
defp dictionary_connection_params() do
Plausible.IngestRepo.config()
|> Enum.map(fn
{:database, database} -> "DB '#{database}'"
{:username, username} -> "USER '#{username}'"
{:password, password} -> "PASSWORD '#{password}'"
_ -> nil
end)
|> Enum.reject(&is_nil/1)
|> Enum.join(" ")
end
defp get_partitions(opts) do
[min_partition, max_partition] = Keyword.get(opts, :partition_range, ["0", "999999"])
{:ok, %{rows: [[partitions, parts]]}} =
run_sql("list-partitions", min_partition: min_partition, max_partition: max_partition)
{partitions, parts, min_partition, max_partition}
end
defp format_duration(seconds) do
seconds
|> Timex.Duration.from_seconds()
|> Timex.format_duration(Timex.Format.Duration.Formatters.Humanized)
end
end

@ -1,188 +0,0 @@
defmodule Plausible.DataMigration.SiteImports do
@moduledoc """
Site imports migration backfilling SiteImport entries for old imports
and alters import end dates to match actual end date of respective import stats.
"""
import Ecto.Query
alias Plausible.ClickhouseRepo
alias Plausible.Imported
alias Plausible.Repo
alias Plausible.Site
def run(opts \\ []) do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
sites_with_imports =
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|> Repo.all(log: false)
sites_count = length(sites_with_imports)
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
for {site, idx} <- Enum.with_index(sites_with_imports) do
site_imports =
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|> Repo.all(log: false)
IO.puts(
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
)
site_imports =
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
IO.puts("Creating legacy site import entry for site ID #{site.id}")
# create legacy entry if there's not one yet
params =
site.imported_data
|> Imported.SiteImport.from_legacy()
|> Map.put(:site_id, site.id)
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
legacy_site_import =
%Imported.SiteImport{}
|> Ecto.Changeset.change(params)
|> insert!(dry_run?)
[legacy_site_import | site_imports]
else
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
site_imports
end
# adjust end date for each site import
for site_import <- site_imports do
IO.puts(
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
)
import_ids =
if site_import.legacy do
[0, site_import.id]
else
[site_import.id]
end
end_date = imported_stats_end_date(site.id, import_ids)
if !end_date do
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."
)
if site_import.legacy do
clear_imported_data(site, dry_run?)
end
delete!(site_import, dry_run?)
else
end_date =
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
end_date
else
IO.puts(
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
)
site_import.end_date
end
site_import
|> Ecto.Changeset.change(end_date: end_date)
|> update!(dry_run?)
IO.puts(
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
)
end
end
IO.puts("Done processing site ID #{site.id}")
end
IO.puts("Finished")
end
# Exposed for testing purposes
@doc false
def imported_stats_end_date(site_id, import_ids) do
[first_schema | schemas] = Imported.schemas()
query =
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
end)
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
if dates != [] do
case Enum.max(dates, Date) do
# no stats for this domain yet
~D[1970-01-01] ->
nil
date ->
date
end
else
nil
end
end
defp insert!(changeset, false = _dry_run?) do
Repo.insert!(changeset)
end
defp insert!(changeset, true = _dry_run?) do
if changeset.valid? do
changeset
|> Ecto.Changeset.change(id: 0)
|> Ecto.Changeset.apply_changes()
else
raise "Invalid insert: #{inspect(changeset)}"
end
end
defp clear_imported_data(site, false = _dry_run?) do
Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
end
defp clear_imported_data(site, true = _dry_run?) do
%{site | imported_data: nil}
end
defp update!(changeset, false = _dry_run?) do
Repo.update!(changeset)
end
defp update!(changeset, true = _dry_run?) do
if changeset.valid? do
Ecto.Changeset.apply_changes(changeset)
else
raise "Invalid update: #{inspect(changeset)}"
end
end
defp delete!(entity, false = _dry_run?) do
Repo.delete!(entity)
end
defp delete!(entity, true = _dry_run?) do
entity
end
defp max_date_query(schema, site_id, import_ids) do
from(q in schema,
where: q.site_id == ^site_id,
where: q.import_id in ^import_ids,
select: %{max_date: fragment("max(?)", q.date)}
)
end
end

@ -1,75 +0,0 @@
defmodule Plausible.DataMigration.VersionedSessions do
@moduledoc """
Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
priv/data_migrations/VersionedSessions/sql
"""
use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
@suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
@versioned_table_engines [
"ReplicatedVersionedCollapsingMergeTree",
"VersionedCollapsingMergeTree"
]
def run(opts \\ []) do
run_exchange? = Keyword.get(opts, :run_exchange?, true)
unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
{:ok, %{rows: partitions}} = run_sql("list-partitions")
partitions = Enum.map(partitions, fn [part] -> part end)
{:ok, %{rows: [[current_table_engine, table_settings]]}} =
run_sql("get-sessions-table-settings")
if Enum.member?(@versioned_table_engines, current_table_engine) do
IO.puts("sessions_v2 table is already versioned, no migration needed")
else
{:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
{:ok, _} =
run_sql("create-sessions-tmp-table",
cluster?: cluster?,
table_settings: table_settings,
unique_suffix: unique_suffix
)
for partition <- partitions do
{:ok, _} = run_sql("attach-partition", partition: partition)
end
if run_exchange? do
run_exchange(cluster?)
end
IO.puts("Migration done!")
end
end
defp run_exchange(cluster?) do
case run_sql("exchange-sessions-tables", cluster?: cluster?) do
{:ok, _} ->
nil
# Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
{:error, %Ch.Error{code: 1}} ->
IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
{:ok, _} =
run_sql("rename-table",
from: "sessions_v2",
to: "sessions_v2_backup",
cluster?: cluster?
)
{:ok, _} =
run_sql("rename-table",
from: "sessions_v2_tmp_versioned",
to: "sessions_v2",
cluster?: cluster?
)
end
end
end

@ -1,371 +0,0 @@
defmodule Plausible.Exports do
@moduledoc """
Contains functions to export data for events and sessions as Zip archives.
"""
require Plausible
import Ecto.Query
@doc """
Renders filename for the Zip archive containing the exported CSV files.
Examples:
iex> archive_filename("plausible.io", ~D[2021-01-01], ~D[2024-12-31])
"plausible_io_20210101_20241231.zip"
iex> archive_filename("Bücher.example", ~D[2021-01-01], ~D[2024-12-31])
"Bücher_example_20210101_20241231.zip"
"""
def archive_filename(domain, min_date, max_date) do
name =
Enum.join(
[
String.replace(domain, ".", "_"),
Calendar.strftime(min_date, "%Y%m%d"),
Calendar.strftime(max_date, "%Y%m%d")
],
"_"
)
name <> ".zip"
end
@doc """
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
tables into the format of `imported_*` tables for a website.
"""
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
%{String.t() => Ecto.Query.t()}
def export_queries(site_id, opts \\ []) do
extname = opts[:extname] || ".csv"
date_range = opts[:date_range]
filename = fn table ->
name =
if date_range do
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
"#{table}_#{first_date}_#{last_date}"
else
table
end
name <> extname
end
%{
filename.("imported_visitors") => export_visitors_q(site_id),
filename.("imported_sources") => export_sources_q(site_id),
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
filename.("imported_pages") => export_pages_q(site_id),
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
filename.("imported_locations") => export_locations_q(site_id),
filename.("imported_devices") => export_devices_q(site_id),
filename.("imported_browsers") => export_browsers_q(site_id),
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
}
end
Plausible.on_full_build do
defp sampled(table) do
Plausible.Stats.Sampling.add_query_hint(from(table))
end
else
defp sampled(table) do
table
end
end
defmacrop date(timestamp) do
quote do
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
end
end
defmacrop visit_duration(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
:visit_duration
)
end
end
defmacrop visitors(t) do
quote do
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
:visitors
)
end
end
defmacrop visits(t) do
quote do
selected_as(sum(unquote(t).sign), :visits)
end
end
defmacrop bounces(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
:bounces
)
end
end
defmacrop pageviews(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
:pageviews
)
end
end
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
def export_visitors_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: selected_as(:date),
select: [
date(s.start),
visitors(s),
pageviews(s),
bounces(s),
visits(s),
visit_duration(s)
]
end
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
def export_sources_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [
selected_as(:date),
selected_as(:source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
s.utm_term
],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.referrer_source, :source),
s.referrer,
s.utm_source,
s.utm_medium,
s.utm_campaign,
s.utm_content,
s.utm_term,
pageviews(s),
visitors(s),
visits(s),
visit_duration(s),
bounces(s)
]
end
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
def export_pages_q(site_id) do
window_q =
from e in sampled("events_v2"),
where: e.site_id == ^site_id,
select: %{
timestamp: e.timestamp,
next_timestamp:
over(fragment("leadInFrame(?)", e.timestamp),
partition_by: e.session_id,
order_by: e.timestamp,
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
),
pathname: e.pathname,
hostname: e.hostname,
name: e.name,
user_id: e.user_id,
session_id: e.session_id,
_sample_factor: fragment("_sample_factor")
}
from e in subquery(window_q),
group_by: [selected_as(:date), e.pathname],
order_by: selected_as(:date),
select: [
date(e.timestamp),
selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page),
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
:visits
),
visitors(e),
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
:pageviews
),
# NOTE: are exits pageviews or any events?
selected_as(
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
:exits
),
selected_as(
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
:time_on_page
)
]
end
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
def export_entry_pages_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.entry_page],
order_by: selected_as(:date),
select: [
date(s.start),
s.entry_page,
visitors(s),
selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:entrances
),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
def export_exit_pages_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.exit_page],
order_by: selected_as(:date),
select: [
date(s.start),
s.exit_page,
visitors(s),
visit_duration(s),
selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:exits
),
bounces(s),
pageviews(s)
]
end
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
def export_locations_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.country_code, :country),
selected_as(s.subdivision1_code, :region),
selected_as(s.city_geoname_id, :city),
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
def export_devices_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.screen_size],
order_by: selected_as(:date),
select: [
date(s.start),
selected_as(s.screen_size, :device),
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
def export_browsers_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.browser, s.browser_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.browser,
s.browser_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
def export_operating_systems_q(site_id) do
from s in sampled("sessions_v2"),
where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
order_by: selected_as(:date),
select: [
date(s.start),
s.operating_system,
s.operating_system_version,
visitors(s),
visits(s),
visit_duration(s),
bounces(s),
pageviews(s)
]
end
@doc """
Creates a streamable Zip archive from the provided (named) Ecto queries.
Example usage:
{:ok, pool} = Ch.start_link(pool_size: 1)
DBConnection.run(pool, fn conn ->
conn
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|> Stream.into(File.stream!("export.zip"))
|> Stream.run()
end)
"""
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
Enumerable.t()
def stream_archive(conn, named_queries, opts \\ []) do
entries =
Enum.map(named_queries, fn {name, query} ->
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
datastream =
conn
|> Ch.stream(sql, params, opts)
|> Stream.map(fn %Ch.Result{data: data} -> data end)
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
end)
Zstream.zip(entries)
end
end

@ -22,9 +22,6 @@ defmodule Plausible.Geo do
* `:edition` - the name of the MaxMind database to be downloaded from MaxMind
servers. Defaults to `GeoLite2-City`.
* `:cache_dir` - if set, the downloaded .mmdb files are cached there across
restarts.
* `:async` - when used, configures the database loading to run
asynchronously.
@ -45,19 +42,17 @@ defmodule Plausible.Geo do
cond do
license_key = opts[:license_key] ->
edition = opts[:edition] || "GeoLite2-City"
maxmind_opts = [license_key: license_key]
loader_opts =
if is_binary(opts[:cache_dir]) do
[
if is_binary(opts[:cache_dir]) do
:ok =
:locus.start_loader(@db, {:maxmind, edition},
license_key: license_key,
database_cache_file:
String.to_charlist(Path.join(opts[:cache_dir], edition <> ".mmdb.gz"))
]
else
[:no_cache]
end
:ok = :locus.start_loader(@db, {:maxmind, edition}, maxmind_opts ++ loader_opts)
)
else
:ok = :locus.start_loader(@db, {:maxmind, edition}, license_key: license_key)
end
path = opts[:path] ->
:ok = :locus.start_loader(@db, path)

@ -80,14 +80,6 @@ defmodule Plausible.Goals do
def find_or_create(_, %{"goal_type" => "page"}), do: {:missing, "page_path"}
def list_revenue_goals(site) do
from(g in Plausible.Goal,
where: g.site_id == ^site.id and not is_nil(g.currency),
select: %{event_name: g.event_name, currency: g.currency}
)
|> Plausible.Repo.all()
end
def for_site(site, opts \\ []) do
site
|> for_site_query(opts)

@ -1,14 +1,10 @@
defmodule Plausible.Google.API do
@moduledoc """
API to Google services.
"""
defmodule Plausible.Google.Api do
alias Plausible.Google.{ReportRequest, HTTP}
use Timex
alias Plausible.Google.HTTP
require Logger
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
@search_console_scope URI.encode_www_form(
"email https://www.googleapis.com/auth/webmasters.readonly"
)
@ -21,46 +17,9 @@ defmodule Plausible.Google.API do
Jason.encode!([site_id, redirect_to])
end
def import_authorize_url(site_id, redirect_to, opts \\ []) do
legacy = Keyword.get(opts, :legacy, true)
def import_authorize_url(site_id, redirect_to) do
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
Jason.encode!([site_id, redirect_to, legacy])
end
def fetch_access_token!(code) do
HTTP.fetch_access_token!(code)
end
def list_properties_and_views(access_token) do
with {:ok, properties} <- Plausible.Google.GA4.API.list_properties(access_token),
{:ok, views} <- Plausible.Google.UA.API.list_views(access_token) do
{:ok, properties ++ views}
end
end
def get_property_or_view(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_property(access_token, property_or_view)
else
Plausible.Google.UA.API.get_view(access_token, property_or_view)
end
end
def get_analytics_start_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_start_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_start_date(access_token, property_or_view)
end
end
def get_analytics_end_date(access_token, property_or_view) do
if property?(property_or_view) do
Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view)
else
Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view)
end
Jason.encode!([site_id, redirect_to])
end
def fetch_verified_properties(auth) do
@ -94,7 +53,157 @@ defmodule Plausible.Google.API do
end
end
def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
@spec list_views(access_token :: String.t()) ::
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
@doc """
Lists Google Analytics views grouped by hostname.
"""
def list_views(access_token) do
case HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
views = Enum.group_by(views, &view_hostname/1, &view_names/1)
{:ok, views}
error ->
error
end
end
defp view_hostname(view) do
case view do
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
_any -> "Others"
end
end
defp view_names(%{"name" => name, "id" => id}) do
{"#{id} - #{name}", id}
end
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
@doc """
Returns a single Google Analytics view if the user has access to it.
"""
def get_view(access_token, lookup_id) do
case list_views(access_token) do
{:ok, views} ->
view =
views
|> Map.values()
|> List.flatten()
|> Enum.find(fn {_name, id} -> id == lookup_id end)
{:ok, view}
{:error, cause} ->
{:error, cause}
end
end
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 7_500
@backoff_factor :timer.seconds(10)
@max_attempts 5
@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
This function fetches Google Analytics reports in batches of #{@per_page} per
request. The batches are then buffered to Clickhouse by the
`Plausible.Google.Buffer` process.
Requests to Google Analytics can fail, and are retried at most
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
importing has finished or `{:error, term()}` when a request to GA failed too
many times.
Useful links:
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(site, date_range, view_id, auth) do
with {:ok, access_token} <- maybe_refresh_token(auth),
:ok <- do_import_analytics(site, date_range, view_id, access_token) do
:ok
else
{:error, cause} ->
Sentry.capture_message("Failed to import from Google Analytics",
extra: %{site: site.domain, error: inspect(cause)}
)
{:error, cause}
end
end
defp do_import_analytics(site, date_range, view_id, access_token) do
{:ok, buffer} = Plausible.Google.Buffer.start_link()
result =
Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok ->
report_request = %ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
case fetch_and_persist(site, report_request, buffer: buffer) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
Plausible.Google.Buffer.flush(buffer)
Plausible.Google.Buffer.stop(buffer)
result
end
@spec fetch_and_persist(Plausible.Site.t(), ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(site, %ReportRequest{} = report_request, opts \\ []) do
buffer_pid = Keyword.get(opts, :buffer)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case HTTP.get_report(report_request) do
{:ok, {rows, next_page_token}} ->
records = Plausible.Imported.from_google_analytics(rows, site.id, report_request.dataset)
:ok = Plausible.Google.Buffer.insert_many(buffer_pid, report_request.dataset, records)
if next_page_token do
fetch_and_persist(
site,
%ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
{:error, cause}
else
Process.sleep(attempt * sleep_time)
fetch_and_persist(site, report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
with true <- needs_to_refresh_token?(auth.expires),
{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
changeset <-
@ -110,7 +219,7 @@ defmodule Plausible.Google.API do
end
end
def maybe_refresh_token({access_token, refresh_token, expires_at}) do
defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
with true <- needs_to_refresh_token?(expires_at),
{:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do
{:ok, new_access_token}
@ -120,8 +229,6 @@ defmodule Plausible.Google.API do
end
end
def property?(value), do: String.starts_with?(value, "properties/")
defp do_refresh_token(refresh_token) do
case HTTP.refresh_auth_token(refresh_token) do
{:ok, %{"access_token" => new_access_token, "expires_in" => expires_in}} ->

@ -1,4 +1,4 @@
defmodule Plausible.Imported.Buffer do
defmodule Plausible.Google.Buffer do
@moduledoc """
This GenServer inserts records into Clickhouse `imported_*` tables. Multiple buffers are
automatically created for each table. Records are flushed when the table buffer reaches the
@ -8,13 +8,12 @@ defmodule Plausible.Imported.Buffer do
use GenServer
require Logger
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, opts)
def start_link do
GenServer.start_link(__MODULE__, nil)
end
def init(opts) do
flush_interval = Keyword.get(opts, :flush_interval_ms, 1000)
{:ok, %{flush_interval: flush_interval, buffers: %{}}}
def init(_opts) do
{:ok, %{buffers: %{}}}
end
@spec insert_many(pid(), term(), [map()]) :: :ok
@ -69,27 +68,27 @@ defmodule Plausible.Imported.Buffer do
def handle_call(:flush_all_buffers, _from, state) do
Enum.each(state.buffers, fn {table_name, records} ->
flush_buffer(records, table_name, state.flush_interval)
flush_buffer(records, table_name)
end)
{:reply, :ok, put_in(state.buffers, %{})}
end
def handle_continue({:flush, table_name}, state) do
flush_buffer(state.buffers[table_name], table_name, state.flush_interval)
flush_buffer(state.buffers[table_name], table_name)
{:noreply, put_in(state.buffers[table_name], [])}
end
defp max_buffer_size do
:plausible
|> Application.fetch_env!(:imported)
|> Application.get_env(:google)
|> Keyword.fetch!(:max_buffer_size)
end
defp flush_buffer(records, table_name, flush_interval) do
defp flush_buffer(records, table_name) do
# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
# sleep call slows down the flushing
Process.sleep(flush_interval)
Process.sleep(1000)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
insert_all(table_name, records)
@ -102,13 +101,13 @@ defmodule Plausible.Imported.Buffer do
Plausible.IngestRepo.insert_all(schema, records)
end
defp table_schema("imported_visitors"), do: Plausible.Imported.Visitor
defp table_schema("imported_sources"), do: Plausible.Imported.Source
defp table_schema("imported_pages"), do: Plausible.Imported.Page
defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage
defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage
defp table_schema("imported_locations"), do: Plausible.Imported.Location
defp table_schema("imported_devices"), do: Plausible.Imported.Device
defp table_schema("imported_browsers"), do: Plausible.Imported.Browser
defp table_schema("imported_operating_systems"), do: Plausible.Imported.OperatingSystem
defp table_schema("imported_visitors"), do: Plausible.Google.ImportedVisitor
defp table_schema("imported_sources"), do: Plausible.Google.ImportedSource
defp table_schema("imported_pages"), do: Plausible.Google.ImportedPage
defp table_schema("imported_entry_pages"), do: Plausible.Google.ImportedEntryPage
defp table_schema("imported_exit_pages"), do: Plausible.Google.ImportedExitPage
defp table_schema("imported_locations"), do: Plausible.Google.ImportedLocation
defp table_schema("imported_devices"), do: Plausible.Google.ImportedDevice
defp table_schema("imported_browsers"), do: Plausible.Google.ImportedBrowser
defp table_schema("imported_operating_systems"), do: Plausible.Google.ImportedOperatingSystem
end

@ -1,152 +0,0 @@
defmodule Plausible.Google.GA4.API do
@moduledoc """
API for Google Analytics 4.
"""
alias Plausible.Google
alias Plausible.Google.GA4
require Logger
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 50_000
@backoff_factor :timer.seconds(10)
@max_attempts 5
def list_properties(access_token) do
case GA4.HTTP.list_accounts_for_user(access_token) do
{:ok, %{"accountSummaries" => accounts}} ->
accounts =
accounts
|> Enum.filter(& &1["propertySummaries"])
|> Enum.map(fn account ->
%{"account" => account_id, "displayName" => account_name} = account
{"#{account_name} (#{account_id})",
Enum.map(account["propertySummaries"], fn property ->
%{"displayName" => property_name, "property" => property_id} = property
{"#{property_name} (#{property_id})", property_id}
end)}
end)
{:ok, accounts}
{:error, cause} ->
{:error, cause}
end
end
def get_property(access_token, lookup_property) do
case GA4.HTTP.get_property(access_token, lookup_property) do
{:ok, property} ->
%{"displayName" => property_name, "name" => property_id, "account" => account_id} =
property
{:ok,
%{
id: property_id,
name: "#{property_name} (#{property_id})",
account_id: account_id
}}
{:error, cause} ->
{:error, cause}
end
end
def get_analytics_start_date(access_token, property) do
GA4.HTTP.get_analytics_start_date(access_token, property)
end
def get_analytics_end_date(access_token, property) do
GA4.HTTP.get_analytics_end_date(access_token, property)
end
def import_analytics(date_range, property, auth, persist_fn) do
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
)
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
do_import_analytics(date_range, property, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, property, access_token, persist_fn) do
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
)
report_request = prepare_request(report_request, date_range, property, access_token)
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case GA4.HTTP.get_report(report_request) do
{:ok, {rows, row_count}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
)
:ok = persist_fn.(report_request.dataset, rows)
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
)
if report_request.offset + @per_page < row_count do
fetch_and_persist(
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
)
{:error, cause}
else
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
)
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp prepare_request(report_request, date_range, property, access_token) do
%GA4.ReportRequest{
report_request
| date_range: date_range,
property: property,
access_token: access_token,
offset: 0,
limit: @per_page
}
end
end

@ -1,244 +0,0 @@
defmodule Plausible.Google.GA4.HTTP do
@moduledoc """
HTTP client implementation for Google Analytics 4 API.
"""
alias Plausible.HTTPClient
require Logger
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
params = %{
requests: [
%{
property: report_request.property,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
metrics: Enum.map(report_request.metrics, &build_metric/1),
orderBys: [
%{
dimension: %{
dimensionName: "date",
orderType: "ALPHANUMERIC"
},
desc: true
}
],
limit: report_request.limit,
offset: report_request.offset
}
]
}
url =
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
response =
HTTPClient.impl().post(
url,
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
row_count <- Map.fetch!(report, "rowCount"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, row_count}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
)
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, reason} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
)
{:error, :request_failed}
end
end
defp build_metric(expression) do
case String.split(expression, " = ") do
[name, expression] ->
%{
name: name,
expression: expression
}
[name] ->
%{name: name}
end
end
defp parse_report_from_response(%{"reports" => [report | _]}) do
{:ok, report}
end
defp parse_report_from_response(body) do
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
Logger.error(
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
defp convert_to_maps(%{
"rows" => rows,
"dimensionHeaders" => dimension_headers,
"metricHeaders" => metric_headers
})
when is_list(rows) do
dimension_headers = Enum.map(dimension_headers, & &1["name"])
metric_headers = Enum.map(metric_headers, & &1["name"])
report =
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
dimension_values = Enum.map(dimensions, & &1["value"])
metric_values = Enum.map(metrics, & &1["value"])
metrics = Enum.zip(metric_headers, metric_values)
dimensions = Enum.zip(dimension_headers, dimension_values)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
{:error, {:invalid_response, response}}
end
def list_accounts_for_user(access_token) do
url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
{:error, :unknown}
end
end
def get_property(access_token, property) do
url = "#{admin_api_url()}/v1beta/#{property}"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
{:error, :not_found}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 property #{property}",
extra: %{error: error}
)
{:error, :unknown}
end
end
@earliest_valid_date "2015-08-14"
def get_analytics_start_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :start)
end
def get_analytics_end_date(access_token, property) do
get_analytics_boundary_date(access_token, property, :end)
end
defp get_analytics_boundary_date(access_token, property, edge) do
descending? = edge == :end
params = %{
requests: [
%{
property: "#{property}",
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "date"}],
metrics: [%{name: "screenPageViews"}],
orderBys: [
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
],
limit: 1
}
]
}
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["rows"] do
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %{reason: :timeout}} ->
{:error, :timeout}
{:error, error} ->
Sentry.capture_message("Error retrieving GA4 #{edge} date",
extra: %{error: error}
)
{:error, :unknown}
end
end
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
end

@ -1,135 +0,0 @@
defmodule Plausible.Google.GA4.ReportRequest do
@moduledoc """
Report request struct for Google Analytics 4 API
"""
defstruct [
:dataset,
:dimensions,
:metrics,
:date_range,
:property,
:access_token,
:offset,
:limit
]
@type t() :: %__MODULE__{
dataset: String.t(),
dimensions: [String.t()],
metrics: [String.t()],
date_range: Date.Range.t(),
property: term(),
access_token: String.t(),
offset: non_neg_integer(),
limit: non_neg_integer()
}
def full_report do
[
%__MODULE__{
dataset: "imported_visitors",
dimensions: ["date"],
metrics: [
"totalUsers",
"screenPageViews",
"bounces = sessions - engagedSessions",
"sessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_sources",
dimensions: [
"date",
"sessionSource",
"sessionMedium",
"sessionCampaignName",
"sessionManualAdContent",
"sessionGoogleAdsKeyword",
"pageReferrer"
],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
},
%__MODULE__{
dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"userEngagementDuration",
"bounces = sessions - engagedSessions"
]
},
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
# %__MODULE__{
# dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"],
# metrics: [
# "totalUsers",
# "sessions",
# "screenPageViews",
# "userEngagementDuration",
# "bounces = sessions - engagedSessions"
# ]
# },
%__MODULE__{
dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_devices",
dimensions: ["date", "deviceCategory"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_browsers",
dimensions: ["date", "browser"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
metrics: [
"screenPageViews",
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
}
]
end
end

@ -2,6 +2,97 @@ defmodule Plausible.Google.HTTP do
require Logger
alias Plausible.HTTPClient
@spec get_report(Plausible.Google.ReportRequest.t()) ::
{:ok, {[map()], String.t() | nil}} | {:error, any()}
def get_report(%Plausible.Google.ReportRequest{} = report_request) do
params = %{
reportRequests: [
%{
viewId: report_request.view_id,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: report_request.page_size,
pageToken: report_request.page_token
}
]
}
response =
HTTPClient.impl().post(
"#{reporting_api_url()}/v4/reports:batchGet",
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
token <- Map.get(report, "nextPageToken"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, token}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, _reason} ->
{:error, :request_failed}
end
end
defp parse_report_from_response(body) do
with %{"reports" => [report | _]} <- body do
{:ok, report}
else
_ ->
Sentry.Context.set_extra_context(%{google_analytics_response: body})
Logger.error(
"Google Analytics: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
end
defp convert_to_maps(%{
"data" => %{} = data,
"columnHeader" => %{
"dimensions" => dimension_headers,
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
}
}) do
metric_headers = Enum.map(metric_headers, & &1["name"])
rows = Map.get(data, "rows", [])
report =
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
metrics = Enum.zip(metric_headers, metrics)
dimensions = Enum.zip(dimension_headers, dimensions)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics_response: response})
{:error, {:invalid_response, response}}
end
def list_sites(access_token) do
url = "#{api_url()}/webmasters/v3/sites"
headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]
@ -22,7 +113,7 @@ defmodule Plausible.Google.HTTP do
end
end
def fetch_access_token!(code) do
def fetch_access_token(code) do
url = "#{api_url()}/oauth2/v4/token"
headers = [{"Content-Type", "application/x-www-form-urlencoded"}]
@ -39,6 +130,24 @@ defmodule Plausible.Google.HTTP do
response.body
end
def list_views_for_user(access_token) do
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
{:error, :unknown}
end
end
def list_stats(access_token, property, date_range, limit, page \\ nil) do
property = URI.encode_www_form(property)
@ -106,9 +215,57 @@ defmodule Plausible.Google.HTTP do
end
end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(view_id, access_token) do
params = %{
reportRequests: [
%{
viewId: view_id,
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "ga:date", histogramBuckets: []}],
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
pageSize: 1
}
]
}
url = "#{reporting_api_url()}/v4/reports:batchGet"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["data"]["rows"] do
[%{"dimensions" => [date_str]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)})
{:error, reason}
end
end
defp config, do: Application.get_env(:plausible, :google)
defp client_id, do: Keyword.fetch!(config(), :client_id)
defp client_secret, do: Keyword.fetch!(config(), :client_secret)
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url)
defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
end

@ -1,8 +1,4 @@
defmodule Plausible.Google.UA.ReportRequest do
@moduledoc """
Report request struct for Universal Analytics API
"""
defmodule Plausible.Google.ReportRequest do
defstruct [
:dataset,
:dimensions,

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save