Improve import stats toggle and `with_imported` flag computation (#3960)

* Check import presence across all imports and not just the first one

Also, simplify imported data toggle rendering to not explicitly
refer to the earliest import source.

* Change imported stats toggle icon in dashboard

* Test `Imported.get_imports_date_range/1`

* Simplify failed UA/GA import email copy
pull/3965/head r2024.04.02-419
Adrian Gruntkowski 3 years ago committed by GitHub
parent 71fe541359
commit 23a3699dd7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -15,6 +15,7 @@ import * as url from '../../util/url'
import classNames from 'classnames';
import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date'
import { isComparisonEnabled } from '../../comparison-input'
import { BarsArrowUpIcon } from '@heroicons/react/20/solid'
const calculateMaximumY = function(dataset) {
const yAxisValues = dataset
@ -286,7 +287,7 @@ class LineGraph extends React.Component {
}
importedNotice() {
if (!this.props.topStatData?.imported_source) return
if (!this.props.topStatData?.imports_exist) return
const isBeforeNativeStats = (date) => {
if (!date) return false
@ -301,18 +302,15 @@ class LineGraph extends React.Component {
const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from)
if (isQueryingImportedPeriod || isComparingImportedPeriod) {
const source = this.props.topStatData.imported_source
const withImported = this.props.topStatData.with_imported;
const strike = withImported ? "" : " line-through"
const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400"
const target = url.setQuery('with_imported', !withImported)
const tip = withImported ? "" : "do not ";
return (
<Link to={target} className="w-4 h-4 mx-2">
<div tooltip={`Stats ${tip}include data imported from ${source}.`} className="cursor-pointer w-4 h-4">
<svg className="absolute dark:text-gray-300 text-gray-700" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<text x="4" y="18" fontSize="24" fill="currentColor" className={"text-gray-700 dark:text-gray-300" + strike}>{source[0].toUpperCase()}</text>
</svg>
<div tooltip={`Stats ${tip}include imported data.`} className="cursor-pointer w-4 h-4">
<BarsArrowUpIcon className={"absolute " + toggleColor} />
</div>
</Link>
)

@ -51,13 +51,13 @@ defmodule Plausible.Imported do
def load_import_data(site) do
complete_import_ids = list_complete_import_ids(site)
earliest_import = get_earliest_import(site) || %{}
dates = get_imports_date_range(site)
%{
site
| import_data_loaded: true,
earliest_import_start_date: Map.get(earliest_import, :start_date),
earliest_import_end_date: Map.get(earliest_import, :end_date),
earliest_import_start_date: dates.start_date,
latest_import_end_date: dates.end_date,
complete_import_ids: complete_import_ids
}
end
@ -110,24 +110,35 @@ defmodule Plausible.Imported do
end
end
@spec get_earliest_import(Site.t()) :: SiteImport.t() | nil
def get_earliest_import(site) do
first_import =
@spec get_imports_date_range(Site.t()) :: %{
start_date: Date.t() | nil,
end_date: Date.t() | nil
}
def get_imports_date_range(site) do
dates =
from(i in SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed(),
order_by: i.start_date,
limit: 1
select: %{start_date: min(i.start_date), end_date: max(i.end_date)}
)
|> Repo.one()
legacy_import =
if site.imported_data && site.imported_data.status == "ok" do
site.imported_data
end
dates = dates || %{start_date: nil, end_date: nil}
[legacy_import, first_import]
|> Enum.reject(&is_nil/1)
|> Enum.min_by(& &1.start_date, Date, fn -> nil end)
if site.imported_data && site.imported_data.status == "ok" do
start_date =
[dates.start_date, site.imported_data.start_date]
|> Enum.reject(&is_nil/1)
|> Enum.min(Date, fn -> nil end)
end_date =
[dates.end_date, site.imported_data.end_date]
|> Enum.reject(&is_nil/1)
|> Enum.max(Date, fn -> nil end)
%{start_date: start_date, end_date: end_date}
else
dates
end
end
@spec delete_imports_for_site(Site.t()) :: :ok

@ -59,7 +59,7 @@ defmodule Plausible.Site do
# `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true
field :earliest_import_end_date, :date, virtual: true
field :latest_import_end_date, :date, virtual: true
field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps()

@ -261,8 +261,8 @@ defmodule Plausible.Stats.Query do
@spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean()
def include_imported?(query, site, requested?) do
cond do
is_nil(site.earliest_import_end_date) -> false
Date.after?(query.date_range.first, site.earliest_import_end_date) -> false
is_nil(site.latest_import_end_date) -> false
Date.after?(query.date_range.first, site.latest_import_end_date) -> false
Enum.any?(query.filters) -> false
query.period == "realtime" -> false
true -> requested?

@ -4,7 +4,6 @@ defmodule PlausibleWeb.Api.StatsController do
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Imported.SiteImport
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
alias PlausibleWeb.Api.Helpers, as: H
@ -68,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController do
* `with_imported` - boolean indicating whether the Google Analytics data
was queried or not.
* `imported_source` - the source of the imported data, when applicable.
Currently only Google Analytics is supported.
* `imports_exist` - boolean indicating whether there are any completed
imports for a given site or not.
* `full_intervals` - map of dates indicating whether the interval has been
cut off by the requested date range or not. For example, if looking at a
@ -87,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController do
"2021-11-01" => true,
"2021-12-01" => false
},
"imported_source" => nil,
"imports_exist" => false,
"interval" => "month",
"labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"],
"plot" => [0, 0, 0, 0],
@ -129,8 +128,6 @@ defmodule PlausibleWeb.Api.StatsController do
present_index = present_index_for(site, query, labels)
full_intervals = build_full_intervals(query, labels)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
plot: plot_timeseries(timeseries_result, metric),
labels: labels,
@ -139,7 +136,7 @@ defmodule PlausibleWeb.Api.StatsController do
present_index: present_index,
interval: query.interval,
with_imported: with_imported?(query, comparison_query),
imported_source: site_import && SiteImport.label(site_import),
imports_exist: site.complete_import_ids != [],
full_intervals: full_intervals
})
else
@ -214,14 +211,12 @@ defmodule PlausibleWeb.Api.StatsController do
{top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query)
site_import = Plausible.Imported.get_earliest_import(site)
json(conn, %{
top_stats: top_stats,
interval: query.interval,
sample_percent: sample_percent,
with_imported: with_imported?(query, comparison_query),
imported_source: site_import && SiteImport.label(site_import),
imports_exist: site.complete_import_ids != [],
comparing_from: comparison_query && comparison_query.date_range.first,
comparing_to: comparison_query && comparison_query.date_range.last,
from: query.date_range.first,

@ -6,22 +6,8 @@
<%= link("Click here", to: @link) %> to view your dashboard.
<% else %>
Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that!
<br /> <br />
This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API.
<br /> <br />
Please log into your Google Analytics account and go to Admin -> Property Settings -> User Analysis. Then make sure that "Enable User Metric in Reporting" is set to OFF. Then try importing again. You can switch the setting back after the import.
<br /> <br /> If you've already changed that setting, then it is something else: <br />
<ul>
<li>
Do you have any stats in the Google Analytics property you tried to import? We only import the stats from your first Google Analytics visitor until your first Plausible Analytics visitor. The import fails if there's no returned data from Google from before you counted your first Plausible visitor.
</li>
<li>
Did you set strict data retention limits in your Google Analytics account? You can check the data retention limit you have set for your property in your Google Analytics admin settings by visiting the "Tracking Info" section and clicking on "Data Retention".
</li>
</ul>
<br />
If all of the above is fine, please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<br /><br />
Please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need.
<%= if full_build?() do %>
<br /> <br />
Please reply to this email to let us know if you're still experiencing issues with the import.

@ -60,6 +60,65 @@ defmodule Plausible.ImportedTest do
end
end
describe "get_imports_date_range/1" do
test "returns empty when there are no imports" do
site = insert(:site)
assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
end
test "returns empty when only incomplete or failed imports are present" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"error"
)
|> Repo.update!()
_import1 = insert(:site_import, site: site, status: :pending)
_import2 = insert(:site_import, site: site, status: :importing)
_import3 = insert(:site_import, site: site, status: :failed)
_rogue_import = insert(:site_import, site: build(:site), status: :completed)
assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site)
end
test "returns start and end dates considering all imports" do
site =
insert(:site)
|> Plausible.Site.start_import(
~D[2020-04-01],
~D[2022-06-22],
"Google Analytics",
"ok"
)
|> Repo.update!()
_import1 =
insert(:site_import,
site: site,
start_date: ~D[2020-04-02],
end_date: ~D[2022-06-22],
status: :completed,
legacy: true
)
_import2 =
insert(:site_import,
site: site,
start_date: ~D[2022-06-22],
end_date: ~D[2024-01-08],
status: :completed
)
assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} =
Imported.get_imports_date_range(site)
end
end
describe "check_dates/3" do
test "crops dates from both ends when overlapping with existing import and native stats" do
site = insert(:site)

@ -93,7 +93,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -142,7 +142,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics"} =
assert %{"plot" => plot, "imports_exist" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 5

@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert plot == [2] ++ List.duplicate(0, 23)
@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
"/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true"
)
assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} =
assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} =
json_response(conn, 200)
assert Enum.count(plot) == 31
@ -1027,7 +1027,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => true
} = json_response(conn, 200)
@ -1084,7 +1084,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => plot,
"comparison_plot" => comparison_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)
@ -1114,7 +1114,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert %{
"plot" => this_week_plot,
"comparison_plot" => last_week_plot,
"imported_source" => "Google Analytics",
"imports_exist" => true,
"with_imported" => false
} = json_response(conn, 200)

Loading…
Cancel
Save