From 23a3699dd77aa39be9052aa724842ea2f95463be Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 2 Apr 2024 12:53:19 +0200 Subject: [PATCH] Improve import stats toggle and `with_imported` flag computation (#3960) * Check import presence across all imports and not just the first one Also, simplify imported data toggle rendering to not explicitly refer to the earliest import source. * Change imported stats toggle icon in dashboard * Test `Imported.get_imports_date_range/1` * Simplify failed UA/GA import email copy --- .../js/dashboard/stats/graph/visitor-graph.js | 12 ++-- lib/plausible/imported.ex | 41 ++++++++----- lib/plausible/site.ex | 2 +- lib/plausible/stats/query.ex | 4 +- .../controllers/api/stats_controller.ex | 15 ++--- .../email/google_analytics_import.html.heex | 18 +----- test/plausible/imported_test.exs | 59 +++++++++++++++++++ .../api/stats_controller/imported_test.exs | 4 +- .../api/stats_controller/main_graph_test.exs | 12 ++-- 9 files changed, 108 insertions(+), 59 deletions(-) diff --git a/assets/js/dashboard/stats/graph/visitor-graph.js b/assets/js/dashboard/stats/graph/visitor-graph.js index e925649c0..966d31d14 100644 --- a/assets/js/dashboard/stats/graph/visitor-graph.js +++ b/assets/js/dashboard/stats/graph/visitor-graph.js @@ -15,6 +15,7 @@ import * as url from '../../util/url' import classNames from 'classnames'; import { monthsBetweenDates, parseNaiveDate, isBefore } from '../../util/date' import { isComparisonEnabled } from '../../comparison-input' +import { BarsArrowUpIcon } from '@heroicons/react/20/solid' const calculateMaximumY = function(dataset) { const yAxisValues = dataset @@ -286,7 +287,7 @@ class LineGraph extends React.Component { } importedNotice() { - if (!this.props.topStatData?.imported_source) return + if (!this.props.topStatData?.imports_exist) return const isBeforeNativeStats = (date) => { if (!date) return false @@ -301,18 +302,15 @@ class LineGraph extends React.Component { const isComparingImportedPeriod = isBeforeNativeStats(this.props.topStatData.comparing_from) if (isQueryingImportedPeriod || isComparingImportedPeriod) { - const source = this.props.topStatData.imported_source const withImported = this.props.topStatData.with_imported; - const strike = withImported ? "" : " line-through" + const toggleColor = withImported ? " dark:text-gray-300 text-gray-700" : " dark:text-gray-500 text-gray-400" const target = url.setQuery('with_imported', !withImported) const tip = withImported ? "" : "do not "; return ( -
- - {source[0].toUpperCase()} - +
+
) diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex index 7c6700683..742886a7b 100644 --- a/lib/plausible/imported.ex +++ b/lib/plausible/imported.ex @@ -51,13 +51,13 @@ defmodule Plausible.Imported do def load_import_data(site) do complete_import_ids = list_complete_import_ids(site) - earliest_import = get_earliest_import(site) || %{} + dates = get_imports_date_range(site) %{ site | import_data_loaded: true, - earliest_import_start_date: Map.get(earliest_import, :start_date), - earliest_import_end_date: Map.get(earliest_import, :end_date), + earliest_import_start_date: dates.start_date, + latest_import_end_date: dates.end_date, complete_import_ids: complete_import_ids } end @@ -110,24 +110,35 @@ defmodule Plausible.Imported do end end - @spec get_earliest_import(Site.t()) :: SiteImport.t() | nil - def get_earliest_import(site) do - first_import = + @spec get_imports_date_range(Site.t()) :: %{ + start_date: Date.t() | nil, + end_date: Date.t() | nil + } + def get_imports_date_range(site) do + dates = from(i in SiteImport, where: i.site_id == ^site.id and i.status == ^SiteImport.completed(), - order_by: i.start_date, - limit: 1 + select: %{start_date: min(i.start_date), end_date: max(i.end_date)} ) |> Repo.one() - legacy_import = - if site.imported_data && site.imported_data.status == "ok" do - site.imported_data - end + dates = dates || %{start_date: nil, end_date: nil} - [legacy_import, first_import] - |> Enum.reject(&is_nil/1) - |> Enum.min_by(& &1.start_date, Date, fn -> nil end) + if site.imported_data && site.imported_data.status == "ok" do + start_date = + [dates.start_date, site.imported_data.start_date] + |> Enum.reject(&is_nil/1) + |> Enum.min(Date, fn -> nil end) + + end_date = + [dates.end_date, site.imported_data.end_date] + |> Enum.reject(&is_nil/1) + |> Enum.max(Date, fn -> nil end) + + %{start_date: start_date, end_date: end_date} + else + dates + end end @spec delete_imports_for_site(Site.t()) :: :ok diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex index 5a4de9f01..6085bb5d0 100644 --- a/lib/plausible/site.ex +++ b/lib/plausible/site.ex @@ -59,7 +59,7 @@ defmodule Plausible.Site do # `site` in `assigns`. field :import_data_loaded, :boolean, default: false, virtual: true field :earliest_import_start_date, :date, virtual: true - field :earliest_import_end_date, :date, virtual: true + field :latest_import_end_date, :date, virtual: true field :complete_import_ids, {:array, :integer}, default: [], virtual: true timestamps() diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 6af57aca8..6a0e3bb35 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -261,8 +261,8 @@ defmodule Plausible.Stats.Query do @spec include_imported?(t(), Plausible.Site.t(), boolean()) :: boolean() def include_imported?(query, site, requested?) do cond do - is_nil(site.earliest_import_end_date) -> false - Date.after?(query.date_range.first, site.earliest_import_end_date) -> false + is_nil(site.latest_import_end_date) -> false + Date.after?(query.date_range.first, site.latest_import_end_date) -> false Enum.any?(query.filters) -> false query.period == "realtime" -> false true -> requested? diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 6b9798d10..c6900f82d 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -4,7 +4,6 @@ defmodule PlausibleWeb.Api.StatsController do use Plausible.Repo use PlausibleWeb.Plugs.ErrorHandler - alias Plausible.Imported.SiteImport alias Plausible.Stats alias Plausible.Stats.{Query, Comparisons} alias PlausibleWeb.Api.Helpers, as: H @@ -68,8 +67,8 @@ defmodule PlausibleWeb.Api.StatsController do * `with_imported` - boolean indicating whether the Google Analytics data was queried or not. - * `imported_source` - the source of the imported data, when applicable. - Currently only Google Analytics is supported. + * `imports_exist` - boolean indicating whether there are any completed + imports for a given site or not. * `full_intervals` - map of dates indicating whether the interval has been cut off by the requested date range or not. For example, if looking at a @@ -87,7 +86,7 @@ defmodule PlausibleWeb.Api.StatsController do "2021-11-01" => true, "2021-12-01" => false }, - "imported_source" => nil, + "imports_exist" => false, "interval" => "month", "labels" => ["2021-09-01", "2021-10-01", "2021-11-01", "2021-12-01"], "plot" => [0, 0, 0, 0], @@ -129,8 +128,6 @@ defmodule PlausibleWeb.Api.StatsController do present_index = present_index_for(site, query, labels) full_intervals = build_full_intervals(query, labels) - site_import = Plausible.Imported.get_earliest_import(site) - json(conn, %{ plot: plot_timeseries(timeseries_result, metric), labels: labels, @@ -139,7 +136,7 @@ defmodule PlausibleWeb.Api.StatsController do present_index: present_index, interval: query.interval, with_imported: with_imported?(query, comparison_query), - imported_source: site_import && SiteImport.label(site_import), + imports_exist: site.complete_import_ids != [], full_intervals: full_intervals }) else @@ -214,14 +211,12 @@ defmodule PlausibleWeb.Api.StatsController do {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query) - site_import = Plausible.Imported.get_earliest_import(site) - json(conn, %{ top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, with_imported: with_imported?(query, comparison_query), - imported_source: site_import && SiteImport.label(site_import), + imports_exist: site.complete_import_ids != [], comparing_from: comparison_query && comparison_query.date_range.first, comparing_to: comparison_query && comparison_query.date_range.last, from: query.date_range.first, diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.heex b/lib/plausible_web/templates/email/google_analytics_import.html.heex index a9879165c..c0369fd65 100644 --- a/lib/plausible_web/templates/email/google_analytics_import.html.heex +++ b/lib/plausible_web/templates/email/google_analytics_import.html.heex @@ -6,22 +6,8 @@ <%= link("Click here", to: @link) %> to view your dashboard. <% else %> Unfortunately, your Google Analytics import for <%= @site_import.site.domain %> did not complete successfully. Sorry about that! -

- This may be due to the user metric setting in your Google Analytics account. We found that changing the user metric setting is necessary to get the correct data out of the Google API. -

- Please log into your Google Analytics account and go to Admin -> Property Settings -> User Analysis. Then make sure that "Enable User Metric in Reporting" is set to OFF. Then try importing again. You can switch the setting back after the import. -

If you've already changed that setting, then it is something else:
- - -
- If all of the above is fine, please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need. +

+ Please try to do the import once again. Sometimes the Google Analytics API just randomly returns empty data. It's intermittent and random. Trying to do the import again may return what you need. <%= if full_build?() do %>

Please reply to this email to let us know if you're still experiencing issues with the import. diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs index e5a6d0ba1..df81f5a41 100644 --- a/test/plausible/imported_test.exs +++ b/test/plausible/imported_test.exs @@ -60,6 +60,65 @@ defmodule Plausible.ImportedTest do end end + describe "get_imports_date_range/1" do + test "returns empty when there are no imports" do + site = insert(:site) + + assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site) + end + + test "returns empty when only incomplete or failed imports are present" do + site = + insert(:site) + |> Plausible.Site.start_import( + ~D[2020-04-01], + ~D[2022-06-22], + "Google Analytics", + "error" + ) + |> Repo.update!() + + _import1 = insert(:site_import, site: site, status: :pending) + _import2 = insert(:site_import, site: site, status: :importing) + _import3 = insert(:site_import, site: site, status: :failed) + _rogue_import = insert(:site_import, site: build(:site), status: :completed) + + assert %{start_date: nil, end_date: nil} = Imported.get_imports_date_range(site) + end + + test "returns start and end dates considering all imports" do + site = + insert(:site) + |> Plausible.Site.start_import( + ~D[2020-04-01], + ~D[2022-06-22], + "Google Analytics", + "ok" + ) + |> Repo.update!() + + _import1 = + insert(:site_import, + site: site, + start_date: ~D[2020-04-02], + end_date: ~D[2022-06-22], + status: :completed, + legacy: true + ) + + _import2 = + insert(:site_import, + site: site, + start_date: ~D[2022-06-22], + end_date: ~D[2024-01-08], + status: :completed + ) + + assert %{start_date: ~D[2020-04-01], end_date: ~D[2024-01-08]} = + Imported.get_imports_date_range(site) + end + end + describe "check_dates/3" do test "crops dates from both ends when overlapping with existing import and native stats" do site = insert(:site) diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs index e7efc4f9a..e2f5c93c3 100644 --- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs @@ -93,7 +93,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" ) - assert %{"plot" => plot, "imported_source" => "Google Analytics"} = + assert %{"plot" => plot, "imports_exist" => true} = json_response(conn, 200) assert Enum.count(plot) == 31 @@ -142,7 +142,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&interval=week" ) - assert %{"plot" => plot, "imported_source" => "Google Analytics"} = + assert %{"plot" => plot, "imports_exist" => true} = json_response(conn, 200) assert Enum.count(plot) == 5 diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index bb74bf531..deeff4885 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -70,7 +70,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "/api/stats/#{site.domain}/main-graph?period=day&date=2021-01-01&with_imported=true" ) - assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} = + assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} = json_response(conn, 200) assert plot == [2] ++ List.duplicate(0, 23) @@ -137,7 +137,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" ) - assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} = + assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} = json_response(conn, 200) assert Enum.count(plot) == 31 @@ -158,7 +158,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" ) - assert %{"plot" => plot, "imported_source" => "Google Analytics", "with_imported" => true} = + assert %{"plot" => plot, "imports_exist" => true, "with_imported" => true} = json_response(conn, 200) assert Enum.count(plot) == 31 @@ -1027,7 +1027,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert %{ "plot" => plot, "comparison_plot" => comparison_plot, - "imported_source" => "Google Analytics", + "imports_exist" => true, "with_imported" => true } = json_response(conn, 200) @@ -1084,7 +1084,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert %{ "plot" => plot, "comparison_plot" => comparison_plot, - "imported_source" => "Google Analytics", + "imports_exist" => true, "with_imported" => false } = json_response(conn, 200) @@ -1114,7 +1114,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert %{ "plot" => this_week_plot, "comparison_plot" => last_week_plot, - "imported_source" => "Google Analytics", + "imports_exist" => true, "with_imported" => false } = json_response(conn, 200)