Merge branch 'master' into update_ua_inspector
commit
0e75c2cc97
@ -0,0 +1,247 @@
|
||||
import React from 'react';
|
||||
import { withRouter } from 'react-router-dom'
|
||||
import Chart from 'chart.js/auto';
|
||||
import { navigateToQuery } from '../../query'
|
||||
import GraphTooltip from './graph-tooltip'
|
||||
import { buildDataSet, METRIC_LABELS, METRIC_FORMATTER } from './graph-util'
|
||||
import dateFormatter from './date-formatter';
|
||||
import FadeIn from '../../fade-in';
|
||||
import classNames from 'classnames';
|
||||
|
||||
const calculateMaximumY = function(dataset) {
|
||||
const yAxisValues = dataset
|
||||
.flatMap((item) => item.data)
|
||||
.map((item) => item || 0)
|
||||
|
||||
if (yAxisValues) {
|
||||
return Math.max(...yAxisValues)
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
class LineGraph extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.regenerateChart = this.regenerateChart.bind(this);
|
||||
this.updateWindowDimensions = this.updateWindowDimensions.bind(this);
|
||||
}
|
||||
|
||||
getGraphMetric() {
|
||||
let metric = this.props.graphData.metric
|
||||
|
||||
if (metric == 'visitors' && this.props.query.filters.goal) {
|
||||
return 'conversions'
|
||||
} else {
|
||||
return metric
|
||||
}
|
||||
}
|
||||
|
||||
regenerateChart() {
|
||||
const { graphData, query } = this.props
|
||||
const metric = this.getGraphMetric()
|
||||
const graphEl = document.getElementById("main-graph-canvas")
|
||||
this.ctx = graphEl.getContext('2d');
|
||||
const dataSet = buildDataSet(graphData.plot, graphData.comparison_plot, graphData.present_index, this.ctx, METRIC_LABELS[metric])
|
||||
|
||||
return new Chart(this.ctx, {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: graphData.labels,
|
||||
datasets: dataSet
|
||||
},
|
||||
options: {
|
||||
animation: false,
|
||||
plugins: {
|
||||
legend: { display: false },
|
||||
tooltip: {
|
||||
enabled: false,
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
position: 'average',
|
||||
external: GraphTooltip(graphData, metric, query)
|
||||
},
|
||||
},
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
onResize: this.updateWindowDimensions,
|
||||
elements: { line: { tension: 0 }, point: { radius: 0 } },
|
||||
onClick: this.maybeHopToHoveredPeriod.bind(this),
|
||||
scale: {
|
||||
ticks: { precision: 0, maxTicksLimit: 8 }
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
ticks: {
|
||||
callback: METRIC_FORMATTER[metric],
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
},
|
||||
grid: {
|
||||
zeroLineColor: 'transparent',
|
||||
drawBorder: false,
|
||||
}
|
||||
},
|
||||
yComparison: {
|
||||
min: 0,
|
||||
suggestedMax: calculateMaximumY(dataSet),
|
||||
display: false,
|
||||
grid: { display: false },
|
||||
},
|
||||
x: {
|
||||
grid: { display: false },
|
||||
ticks: {
|
||||
callback: function(val, _index, _ticks) {
|
||||
if (this.getLabelForValue(val) == "__blank__") return ""
|
||||
|
||||
const hasMultipleYears =
|
||||
graphData.labels
|
||||
.filter((date) => typeof date === 'string')
|
||||
.map(date => date.split('-')[0])
|
||||
.filter((value, index, list) => list.indexOf(value) === index)
|
||||
.length > 1
|
||||
|
||||
if (graphData.interval === 'hour' && query.period !== 'day') {
|
||||
const date = dateFormatter({
|
||||
interval: "date",
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
const hour = dateFormatter({
|
||||
interval: graphData.interval,
|
||||
longForm: false,
|
||||
period: query.period,
|
||||
shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
|
||||
// Returns a combination of date and hour. This is because
|
||||
// small intervals like hour may return multiple days
|
||||
// depending on the query period.
|
||||
return `${date}, ${hour}`
|
||||
}
|
||||
|
||||
if (graphData.interval === 'minute' && query.period !== 'realtime') {
|
||||
return dateFormatter({
|
||||
interval: "hour", longForm: false, period: query.period,
|
||||
})(this.getLabelForValue(val))
|
||||
}
|
||||
|
||||
return dateFormatter({
|
||||
interval: graphData.interval, longForm: false, period: query.period, shouldShowYear: hasMultipleYears,
|
||||
})(this.getLabelForValue(val))
|
||||
},
|
||||
color: this.props.darkTheme ? 'rgb(243, 244, 246)' : undefined
|
||||
}
|
||||
}
|
||||
},
|
||||
interaction: {
|
||||
mode: 'index',
|
||||
intersect: false,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
repositionTooltip(e) {
|
||||
const tooltipEl = document.getElementById('chartjs-tooltip');
|
||||
if (tooltipEl && window.innerWidth >= 768) {
|
||||
if (e.clientX > 0.66 * window.innerWidth) {
|
||||
tooltipEl.style.right = (window.innerWidth - e.clientX) + window.pageXOffset + 'px'
|
||||
tooltipEl.style.left = null;
|
||||
} else {
|
||||
tooltipEl.style.right = null;
|
||||
tooltipEl.style.left = e.clientX + window.pageXOffset + 'px'
|
||||
}
|
||||
tooltipEl.style.top = e.clientY + window.pageYOffset + 'px'
|
||||
tooltipEl.style.opacity = 1;
|
||||
}
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
if (this.props.graphData) {
|
||||
this.chart = this.regenerateChart();
|
||||
}
|
||||
window.addEventListener('mousemove', this.repositionTooltip);
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps) {
|
||||
const { graphData, darkTheme } = this.props;
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
|
||||
if (
|
||||
graphData !== prevProps.graphData ||
|
||||
darkTheme !== prevProps.darkTheme
|
||||
) {
|
||||
|
||||
if (graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
this.chart = this.regenerateChart();
|
||||
this.chart.update();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
if (!graphData) {
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
|
||||
if (tooltip) {
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
// Ensure that the tooltip doesn't hang around when we are loading more data
|
||||
const tooltip = document.getElementById('chartjs-tooltip');
|
||||
if (tooltip) {
|
||||
tooltip.style.opacity = 0;
|
||||
tooltip.style.display = 'none';
|
||||
}
|
||||
window.removeEventListener('mousemove', this.repositionTooltip)
|
||||
}
|
||||
|
||||
/**
|
||||
* The current ticks' limits are set to treat iPad (regular/Mini/Pro) as a regular screen.
|
||||
* @param {*} chart - The chart instance.
|
||||
* @param {*} dimensions - An object containing the new dimensions *of the chart.*
|
||||
*/
|
||||
updateWindowDimensions(chart, dimensions) {
|
||||
chart.options.scales.x.ticks.maxTicksLimit = dimensions.width < 720 ? 5 : 8
|
||||
}
|
||||
|
||||
maybeHopToHoveredPeriod(e) {
|
||||
const element = this.chart.getElementsAtEventForMode(e, 'index', { intersect: false })[0]
|
||||
const date = this.props.graphData.labels[element.index] || this.props.graphData.comparison_labels[element.index]
|
||||
|
||||
if (this.props.graphData.interval === 'month') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'month', date })
|
||||
} else if (this.props.graphData.interval === 'date') {
|
||||
navigateToQuery(this.props.history, this.props.query, { period: 'day', date })
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
const { graphData } = this.props
|
||||
const canvasClass = classNames('mt-4 select-none', { 'cursor-pointer': !['minute', 'hour'].includes(graphData?.interval) })
|
||||
|
||||
return (
|
||||
<FadeIn show={graphData}>
|
||||
<div className="relative h-96 print:h-auto print:pb-8 w-full z-0">
|
||||
<canvas id="main-graph-canvas" className={canvasClass}></canvas>
|
||||
</div>
|
||||
</FadeIn>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default withRouter(LineGraph)
|
||||
@ -0,0 +1,17 @@
|
||||
import React from "react"
|
||||
|
||||
export default function SamplingNotice({topStatData}) {
|
||||
const samplePercent = topStatData?.samplePercent
|
||||
|
||||
if (samplePercent && samplePercent < 100) {
|
||||
return (
|
||||
<div tooltip={`Stats based on a ${samplePercent}% sample of all visitors`} className="cursor-pointer w-4 h-4 mx-2">
|
||||
<svg className="absolute w-4 h-4 dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
</div>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
import React, { useState } from "react"
|
||||
import * as api from '../../api'
|
||||
import { getCurrentInterval } from "./interval-picker"
|
||||
|
||||
export default function StatsExport({site, query}) {
|
||||
const [exporting, setExporting] = useState(false)
|
||||
|
||||
function startExport() {
|
||||
setExporting(true)
|
||||
document.cookie = "exporting="
|
||||
pollExportReady()
|
||||
}
|
||||
|
||||
function pollExportReady() {
|
||||
if (document.cookie.includes('exporting')) {
|
||||
setTimeout(pollExportReady, 1000)
|
||||
} else {
|
||||
setExporting(false)
|
||||
}
|
||||
}
|
||||
|
||||
function renderLoading() {
|
||||
return (
|
||||
<svg className="animate-spin h-4 w-4 text-indigo-500" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
||||
<path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function renderExportLink() {
|
||||
const interval = getCurrentInterval(site, query)
|
||||
const queryParams = api.serializeQuery(query, [{ interval }])
|
||||
const endpoint = `/${encodeURIComponent(site.domain)}/export${queryParams}`
|
||||
|
||||
return (
|
||||
<a href={endpoint} download onClick={startExport}>
|
||||
<svg className="absolute text-gray-700 feather dark:text-gray-300 hover:text-indigo-600 dark:hover:text-indigo-600" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<path d="M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4"></path>
|
||||
<polyline points="7 10 12 15 17 10"></polyline>
|
||||
<line x1="12" y1="15" x2="12" y2="3"></line>
|
||||
</svg>
|
||||
</a>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-4 h-4 mx-2">
|
||||
{exporting && renderLoading()}
|
||||
{!exporting && renderExportLink()}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
import React from "react"
|
||||
import { parseNaiveDate, isBefore } from '../../util/date'
|
||||
import { Link } from 'react-router-dom'
|
||||
import * as url from '../../util/url'
|
||||
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
|
||||
|
||||
export default function WithImportedSwitch({site, topStatData}) {
|
||||
if (!topStatData?.imports_exist) {
|
||||
return null
|
||||
}
|
||||
|
||||
function isBeforeNativeStats(date) {
|
||||
if (!date) return false
|
||||
|
||||
const nativeStatsBegin = parseNaiveDate(site.nativeStatsBegin)
|
||||
const parsedDate = parseNaiveDate(date)
|
||||
|
||||
return isBefore(parsedDate, nativeStatsBegin, "day")
|
||||
}
|
||||
|
||||
const isQueryingImportedPeriod = isBeforeNativeStats(topStatData.from)
|
||||
const isComparingImportedPeriod = isBeforeNativeStats(topStatData.comparing_from)
|
||||
|
||||
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
|
||||
const withImported = topStatData.with_imported;
|
||||
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
|
||||
const target = url.setQuery('with_imported', !withImported)
|
||||
const tip = withImported ? "" : "do not ";
|
||||
|
||||
return (
|
||||
<Link to={target} className="w-4 h-4 mx-2">
|
||||
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
|
||||
<BarsArrowUpIcon className={"absolute " + toggleColor} />
|
||||
</div>
|
||||
</Link>
|
||||
)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"kind": "analyticsData#batchRunReports",
|
||||
"reports": [
|
||||
{
|
||||
"dimensionHeaders": [
|
||||
{
|
||||
"name": "date"
|
||||
}
|
||||
],
|
||||
"kind": "analyticsData#runReport",
|
||||
"metadata": {
|
||||
"currencyCode": "USD",
|
||||
"timeZone": "Europe/Warsaw"
|
||||
},
|
||||
"metricHeaders": [
|
||||
{
|
||||
"name": "screenPageViews",
|
||||
"type": "TYPE_INTEGER"
|
||||
}
|
||||
],
|
||||
"rowCount": 3,
|
||||
"rows": [
|
||||
{
|
||||
"dimensionValues": [
|
||||
{
|
||||
"value": "20240302"
|
||||
}
|
||||
],
|
||||
"metricValues": [
|
||||
{
|
||||
"value": "13"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20200421"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
{
|
||||
"reports": [
|
||||
{
|
||||
"columnHeader": {
|
||||
"dimensions": [
|
||||
"ga:date"
|
||||
],
|
||||
"metricHeader": {
|
||||
"metricHeaderEntries": [
|
||||
{
|
||||
"name": "ga:pageviews",
|
||||
"type": "INTEGER"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"isDataGolden": true,
|
||||
"rowCount": 849,
|
||||
"rows": [
|
||||
{
|
||||
"dimensions": [
|
||||
"20170118"
|
||||
],
|
||||
"metrics": [
|
||||
{
|
||||
"values": [
|
||||
"37"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nextPageToken": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,171 @@
|
||||
defmodule Plausible.DataMigration.SiteImports do
|
||||
@moduledoc """
|
||||
Site imports migration backfilling SiteImport entries for old imports
|
||||
and alters import end dates to match actual end date of respective import stats.
|
||||
"""
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Plausible.ClickhouseRepo
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
def run(opts \\ []) do
|
||||
dry_run? = Keyword.get(opts, :dry_run?, true)
|
||||
|
||||
site_import_query =
|
||||
from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
|
||||
|
||||
sites_with_imports =
|
||||
from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
|
||||
|> Repo.all(log: false)
|
||||
|
||||
sites_count = length(sites_with_imports)
|
||||
|
||||
IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
|
||||
|
||||
for {site, idx} <- Enum.with_index(sites_with_imports) do
|
||||
site_imports =
|
||||
from(i in Imported.SiteImport, where: i.site_id == ^site.id)
|
||||
|> Repo.all(log: false)
|
||||
|
||||
IO.puts(
|
||||
"Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
|
||||
)
|
||||
|
||||
site_imports =
|
||||
if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
|
||||
IO.puts("Creating legacy site import entry for site ID #{site.id}")
|
||||
|
||||
# create legacy entry if there's not one yet
|
||||
params =
|
||||
site.imported_data
|
||||
|> Imported.SiteImport.from_legacy()
|
||||
|> Map.put(:site_id, site.id)
|
||||
|> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
|
||||
|
||||
legacy_site_import =
|
||||
%Imported.SiteImport{}
|
||||
|> Ecto.Changeset.change(params)
|
||||
|> insert!(dry_run?)
|
||||
|
||||
[legacy_site_import | site_imports]
|
||||
else
|
||||
IO.puts("Legacy site import entry for site ID #{site.id} already exists")
|
||||
|
||||
site_imports
|
||||
end
|
||||
|
||||
# adjust end date for each site import
|
||||
for site_import <- site_imports do
|
||||
IO.puts(
|
||||
"Adjusting end date for site import #{site_import.id} (site ID #{site.id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
|
||||
)
|
||||
|
||||
import_ids =
|
||||
if site_import.legacy do
|
||||
[0, site_import.id]
|
||||
else
|
||||
[site_import.id]
|
||||
end
|
||||
|
||||
end_date = imported_stats_end_date(site.id, import_ids)
|
||||
|
||||
end_date =
|
||||
if !end_date do
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Setting end date to minimum."
|
||||
)
|
||||
|
||||
Date.add(site_import.start_date, 2)
|
||||
else
|
||||
end_date
|
||||
end
|
||||
|
||||
end_date =
|
||||
if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do
|
||||
end_date
|
||||
else
|
||||
IO.puts(
|
||||
"Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."
|
||||
)
|
||||
|
||||
site_import.end_date
|
||||
end
|
||||
|
||||
site_import
|
||||
|> Ecto.Changeset.change(end_date: end_date)
|
||||
|> update!(dry_run?)
|
||||
|
||||
IO.puts(
|
||||
"End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"
|
||||
)
|
||||
end
|
||||
|
||||
IO.puts("Done processing site ID #{site.id}")
|
||||
end
|
||||
|
||||
IO.puts("Finished")
|
||||
end
|
||||
|
||||
# Exposed for testing purposes
|
||||
@doc false
|
||||
def imported_stats_end_date(site_id, import_ids) do
|
||||
[first_schema | schemas] = Imported.schemas()
|
||||
|
||||
query =
|
||||
Enum.reduce(schemas, max_date_query(first_schema, site_id, import_ids), fn schema, query ->
|
||||
from(s in subquery(union_all(query, ^max_date_query(schema, site_id, import_ids))))
|
||||
end)
|
||||
|
||||
dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
|
||||
|
||||
if dates != [] do
|
||||
case Enum.max(dates, Date) do
|
||||
# no stats for this domain yet
|
||||
~D[1970-01-01] ->
|
||||
nil
|
||||
|
||||
date ->
|
||||
date
|
||||
end
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
defp insert!(changeset, false = _dry_run?) do
|
||||
Repo.insert!(changeset)
|
||||
end
|
||||
|
||||
defp insert!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
changeset
|
||||
|> Ecto.Changeset.change(id: 0)
|
||||
|> Ecto.Changeset.apply_changes()
|
||||
else
|
||||
raise "Invalid insert: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp update!(changeset, false = _dry_run?) do
|
||||
Repo.update!(changeset)
|
||||
end
|
||||
|
||||
defp update!(changeset, true = _dry_run?) do
|
||||
if changeset.valid? do
|
||||
Ecto.Changeset.apply_changes(changeset)
|
||||
else
|
||||
raise "Invalid update: #{inspect(changeset)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp max_date_query(schema, site_id, import_ids) do
|
||||
from(q in schema,
|
||||
where: q.site_id == ^site_id,
|
||||
where: q.import_id in ^import_ids,
|
||||
select: %{max_date: fragment("max(?)", q.date)}
|
||||
)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,111 @@
|
||||
defmodule Plausible.Stats.TableDecider do
|
||||
@moduledoc """
|
||||
This module contains logic for deciding which tables need to be queried given a query
|
||||
and metrics, with the purpose of reducing the number of queries and JOINs needed to perform.
|
||||
"""
|
||||
|
||||
import Enum, only: [empty?: 1]
|
||||
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
def events_join_sessions?(query) do
|
||||
Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))
|
||||
end
|
||||
|
||||
def partition_metrics(metrics, query, breakdown_property \\ nil) do
|
||||
%{
|
||||
event: event_only_metrics,
|
||||
session: session_only_metrics,
|
||||
either: either_metrics,
|
||||
other: other_metrics,
|
||||
sample_percent: sample_percent
|
||||
} =
|
||||
partition(metrics, query, &metric_partitioner/2)
|
||||
|
||||
# Treat breakdown property as yet another filter
|
||||
filters =
|
||||
if breakdown_property do
|
||||
Map.put(query.filters, breakdown_property, nil)
|
||||
else
|
||||
query.filters
|
||||
end
|
||||
|
||||
%{event: event_only_filters, session: session_only_filters} =
|
||||
partition(filters, query, &filters_partitioner/2)
|
||||
|
||||
cond do
|
||||
# Only one table needs to be queried
|
||||
empty?(event_only_metrics) && empty?(event_only_filters) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) && empty?(session_only_filters) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Filters on both events and sessions, but only one kind of metric
|
||||
empty?(event_only_metrics) ->
|
||||
{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
|
||||
empty?(session_only_metrics) ->
|
||||
{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}
|
||||
|
||||
# Default: prefer sessions
|
||||
true ->
|
||||
{event_only_metrics ++ sample_percent,
|
||||
session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}
|
||||
end
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, :conversion_rate), do: :event
|
||||
defp metric_partitioner(_, :average_revenue), do: :event
|
||||
defp metric_partitioner(_, :total_revenue), do: :event
|
||||
defp metric_partitioner(_, :pageviews), do: :event
|
||||
defp metric_partitioner(_, :events), do: :event
|
||||
defp metric_partitioner(_, :bounce_rate), do: :session
|
||||
defp metric_partitioner(_, :visit_duration), do: :session
|
||||
defp metric_partitioner(_, :views_per_visit), do: :session
|
||||
|
||||
# Metrics which used to only be queried from one table but can be calculated from either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visits), do: :either
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: true}, :visitors), do: :either
|
||||
|
||||
defp metric_partitioner(_, :visits), do: :session
|
||||
defp metric_partitioner(_, :visitors), do: :event
|
||||
# Calculated metrics - handled on callsite separately from other metrics.
|
||||
defp metric_partitioner(_, :time_on_page), do: :other
|
||||
defp metric_partitioner(_, :total_visitors), do: :other
|
||||
defp metric_partitioner(_, :percentage), do: :other
|
||||
# Sample percentage is included in both tables if queried.
|
||||
defp metric_partitioner(_, :sample_percent), do: :sample_percent
|
||||
|
||||
defp metric_partitioner(%Query{experimental_reduced_joins?: false}, unknown) do
|
||||
raise ArgumentError, "Metric #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp metric_partitioner(_, _), do: :either
|
||||
|
||||
defp filters_partitioner(_, {"event:" <> _, _}), do: :event
|
||||
defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
|
||||
defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
|
||||
do: :either
|
||||
|
||||
defp filters_partitioner(_, {"visit:" <> _, _}),
|
||||
do: :session
|
||||
|
||||
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
|
||||
raise ArgumentError, "Filter #{unknown} not supported without experimental_reduced_joins?"
|
||||
end
|
||||
|
||||
defp filters_partitioner(_, _), do: :either
|
||||
|
||||
@default %{event: [], session: [], either: [], other: [], sample_percent: []}
|
||||
defp partition(values, query, partitioner) do
|
||||
Enum.reduce(values, @default, fn value, acc ->
|
||||
key = partitioner.(query, value)
|
||||
Map.put(acc, key, Map.fetch!(acc, key) ++ [value])
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,35 @@
|
||||
defmodule Plausible.IngestRepo.Migrations.AddMetricsToImportedTables do
|
||||
use Ecto.Migration
|
||||
|
||||
@add_pageviews_for [
|
||||
"imported_browsers",
|
||||
"imported_devices",
|
||||
"imported_entry_pages",
|
||||
"imported_exit_pages",
|
||||
"imported_locations",
|
||||
"imported_operating_systems",
|
||||
"imported_sources"
|
||||
]
|
||||
|
||||
def up do
|
||||
for table <- @add_pageviews_for do
|
||||
add_column(table, "pageviews", "UInt64")
|
||||
end
|
||||
|
||||
add_column("imported_browsers", "browser_version", "String")
|
||||
add_column("imported_exit_pages", "bounces", "UInt32")
|
||||
add_column("imported_exit_pages", "visit_duration", "UInt64")
|
||||
add_column("imported_operating_systems", "operating_system_version", "String")
|
||||
add_column("imported_pages", "visits", "UInt64")
|
||||
add_column("imported_sources", "referrer", "String")
|
||||
add_column("imported_sources", "utm_source", "String")
|
||||
end
|
||||
|
||||
defp add_column(table, column, type) do
|
||||
execute """
|
||||
ALTER TABLE #{table}
|
||||
#{Plausible.MigrationUtils.on_cluster_statement(table)}
|
||||
ADD COLUMN #{column} #{type}
|
||||
"""
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,19 @@
|
||||
defmodule Plausible.ClickhouseRepo.Migrations.HostnamesInSessions do
|
||||
use Ecto.Migration
|
||||
|
||||
def up do
|
||||
execute """
|
||||
ALTER TABLE sessions_v2
|
||||
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
|
||||
ADD COLUMN exit_page_hostname String CODEC(ZSTD(3))
|
||||
"""
|
||||
end
|
||||
|
||||
def down do
|
||||
execute """
|
||||
ALTER TABLE sessions_v2
|
||||
#{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")}
|
||||
DROP COLUMN exit_page_hostname
|
||||
"""
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,188 @@
|
||||
defmodule Plausible.DataMigration.SiteImportsTest do
|
||||
use Plausible.DataCase, async: true
|
||||
|
||||
import ExUnit.CaptureIO
|
||||
|
||||
alias Plausible.DataMigration.SiteImports
|
||||
alias Plausible.Imported
|
||||
alias Plausible.Repo
|
||||
alias Plausible.Site
|
||||
|
||||
describe "run/1" do
|
||||
test "runs for empty dataset" do
|
||||
dry_run_output =
|
||||
capture_io(fn ->
|
||||
assert :ok = SiteImports.run()
|
||||
end)
|
||||
|
||||
assert dry_run_output =~ "Processing 0 sites"
|
||||
assert dry_run_output =~ "DRY RUN: true"
|
||||
|
||||
real_run_output =
|
||||
capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end)
|
||||
|
||||
assert real_run_output =~ "Processing 0 sites"
|
||||
assert real_run_output =~ "DRY RUN: false"
|
||||
end
|
||||
|
||||
test "adds site import entry when it's missing and adjusts end date" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-07])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id > 0
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-07]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
|
||||
test "runs in dry mode without making any persistent changes" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-07])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run()
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)
|
||||
assert id == 0
|
||||
end
|
||||
|
||||
test "does not set end date to latter than the current one" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
populate_stats(site, 0, [
|
||||
build(:imported_visitors, date: ~D[2021-01-10])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id > 0
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-08]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
|
||||
test "leaves site and imports unchanged if everything fits" do
|
||||
site =
|
||||
insert(:site)
|
||||
|> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
|
||||
|> Repo.update!()
|
||||
|
||||
existing_import =
|
||||
insert(:site_import,
|
||||
site: site,
|
||||
start_date: ~D[2021-01-02],
|
||||
end_date: ~D[2021-01-08],
|
||||
status: :completed,
|
||||
legacy: true
|
||||
)
|
||||
|
||||
populate_stats(site, existing_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-08])
|
||||
])
|
||||
|
||||
assert capture_io(fn ->
|
||||
assert :ok = SiteImports.run(dry_run?: false)
|
||||
end) =~ "Processing 1 sites"
|
||||
|
||||
site = Repo.reload!(site)
|
||||
|
||||
assert [%{id: id, legacy: true} = site_import] = Imported.list_all_imports(site)
|
||||
assert id == existing_import.id
|
||||
assert site_import.start_date == site.imported_data.start_date
|
||||
assert site_import.end_date == ~D[2021-01-08]
|
||||
assert site_import.source == :universal_analytics
|
||||
end
|
||||
end
|
||||
|
||||
describe "imported_stats_end_date/1" do
|
||||
test "returns nil when there are no stats" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) == nil
|
||||
end
|
||||
|
||||
test "returns date when there are stats recorded in one imported table" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-01])
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
~D[2021-01-01]
|
||||
end
|
||||
|
||||
test "returns max date across all imported tables" do
|
||||
site_import = insert(:site_import)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(:imported_visitors, date: ~D[2021-01-01]),
|
||||
build(:imported_visitors, date: ~D[2021-01-07]),
|
||||
build(:imported_sources, date: ~D[2021-01-01]),
|
||||
build(:imported_sources, date: ~D[2021-01-08]),
|
||||
build(:imported_entry_pages, date: ~D[2021-01-02]),
|
||||
build(:imported_entry_pages, date: ~D[2021-02-11]),
|
||||
build(:imported_exit_pages, date: ~D[2021-01-01]),
|
||||
build(:imported_exit_pages, date: ~D[2021-01-08]),
|
||||
build(:imported_locations, date: ~D[2021-01-01]),
|
||||
build(:imported_locations, date: ~D[2021-01-08]),
|
||||
build(:imported_devices, date: ~D[2021-01-01]),
|
||||
build(:imported_devices, date: ~D[2021-01-08]),
|
||||
build(:imported_browsers, date: ~D[2021-01-01]),
|
||||
build(:imported_browsers, date: ~D[2021-01-08]),
|
||||
build(:imported_operating_systems, date: ~D[2021-01-01]),
|
||||
build(:imported_operating_systems, date: ~D[2021-01-08])
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
~D[2021-02-11]
|
||||
end
|
||||
|
||||
test "considers all imported tables" do
|
||||
date = ~D[2021-01-11]
|
||||
|
||||
for {table, idx} <- Enum.with_index(Imported.tables()) do
|
||||
site_import = insert(:site_import)
|
||||
end_date = Date.add(date, idx)
|
||||
|
||||
populate_stats(site_import.site, site_import.id, [
|
||||
build(String.to_atom(table), date: date),
|
||||
build(String.to_atom(table), date: end_date)
|
||||
])
|
||||
|
||||
assert SiteImports.imported_stats_end_date(site_import.site_id, [site_import.id]) ==
|
||||
end_date
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,164 @@
|
||||
defmodule Plausible.Stats.TableDeciderTest do
|
||||
use Plausible.DataCase, async: true
|
||||
alias Plausible.Stats.Query
|
||||
|
||||
import Plausible.Stats.TableDecider
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins disabled" do
|
||||
assert not events_join_sessions?(make_query(false, %{}))
|
||||
assert not events_join_sessions?(make_query(false, %{name: "pageview"}))
|
||||
assert events_join_sessions?(make_query(false, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(false, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(false, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
test "events_join_sessions? with experimental_reduced_joins enabled" do
|
||||
assert not events_join_sessions?(make_query(true, %{}))
|
||||
assert not events_join_sessions?(make_query(true, %{name: "pageview"}))
|
||||
assert not events_join_sessions?(make_query(true, %{source: "Google"}))
|
||||
assert events_join_sessions?(make_query(true, %{entry_page: "/"}))
|
||||
assert events_join_sessions?(make_query(true, %{exit_page: "/"}))
|
||||
end
|
||||
|
||||
describe "partition_metrics" do
|
||||
test "with no metrics or filters" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([], query) == {[], [], []}
|
||||
end
|
||||
|
||||
test "session-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :views_per_visit], query) ==
|
||||
{[], [:bounce_rate, :views_per_visit], []}
|
||||
end
|
||||
|
||||
test "event-only metrics accordingly" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, event-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "filters from both, session-only metrics" do
|
||||
query = make_query(false, %{name: "pageview", source: "Google"})
|
||||
|
||||
assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}
|
||||
end
|
||||
|
||||
test "session filters but no session metrics" do
|
||||
query = make_query(false, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}
|
||||
end
|
||||
|
||||
test "sample_percent is added to both types of metrics" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [], []}
|
||||
|
||||
assert partition_metrics([:bounce_rate, :sample_percent], query) ==
|
||||
{[], [:bounce_rate, :sample_percent], []}
|
||||
|
||||
assert partition_metrics([:total_revenue, :bounce_rate, :sample_percent], query) ==
|
||||
{[:total_revenue, :sample_percent], [:bounce_rate, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "other metrics put in its own result" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==
|
||||
{[], [], [:time_on_page, :percentage, :total_visitors]}
|
||||
end
|
||||
|
||||
test "raises if unknown metric" do
|
||||
query = make_query(false, %{})
|
||||
|
||||
assert_raise ArgumentError, fn ->
|
||||
partition_metrics([:foobar], query)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "partition_metrics with experimental_reduced_joins enabled" do
|
||||
test "metrics that can be calculated on either when event-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:total_revenue, :visitors], query) ==
|
||||
{[:total_revenue, :visitors], [], []}
|
||||
|
||||
assert partition_metrics([:pageviews, :visits], query) == {[:pageviews, :visits], [], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either when session-only metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :visitors], query) ==
|
||||
{[], [:bounce_rate, :visitors], []}
|
||||
|
||||
assert partition_metrics([:visit_duration, :visits], query) ==
|
||||
{[], [:visit_duration, :visits], []}
|
||||
end
|
||||
|
||||
test "metrics that can be calculated on either are biased to sessions" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==
|
||||
{[:total_revenue], [:bounce_rate, :visitors], []}
|
||||
end
|
||||
|
||||
test "sample_percent is handled with either metrics" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors, :sample_percent], query) ==
|
||||
{[], [:visitors, :sample_percent], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events" do
|
||||
query = make_query(true, %{name: "pageview"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[:visitors], [], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, but filtering on events and sessions" do
|
||||
query = make_query(true, %{name: "pageview", exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on either" do
|
||||
query = make_query(true, %{source: "Google"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "metric can be calculated on either, filtering on sessions" do
|
||||
query = make_query(true, %{exit_page: "/"})
|
||||
|
||||
assert partition_metrics([:visitors], query) == {[], [:visitors], []}
|
||||
end
|
||||
|
||||
test "breakdown value leans metric" do
|
||||
query = make_query(true, %{})
|
||||
|
||||
assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}
|
||||
assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}
|
||||
assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}
|
||||
end
|
||||
end
|
||||
|
||||
defp make_query(experimental_reduced_joins?, filters) do
|
||||
Query.from(build(:site), %{
|
||||
"experimental_reduced_joins" => to_string(experimental_reduced_joins?),
|
||||
"filters" => Jason.encode!(filters)
|
||||
})
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue