diff --git a/fixture/ga4_end_date.json b/fixture/ga4_end_date.json new file mode 100644 index 000000000..bd066f006 --- /dev/null +++ b/fixture/ga4_end_date.json @@ -0,0 +1,38 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240302" + } + ], + "metricValues": [ + { + "value": "13" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga_end_date.json b/fixture/ga_end_date.json new file mode 100644 index 000000000..68d02b4e7 --- /dev/null +++ b/fixture/ga_end_date.json @@ -0,0 +1,38 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": [ + "ga:date" + ], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga:pageviews", + "type": "INTEGER" + } + ] + } + }, + "data": { + "isDataGolden": true, + "rowCount": 849, + "rows": [ + { + "dimensions": [ + "20200421" + ], + "metrics": [ + { + "values": [ + "37" + ] + } + ] + } + ] + }, + "nextPageToken": "1" + } + ] +} \ No newline at end of file diff --git a/fixture/ga_start_date_later.json b/fixture/ga_start_date_later.json new file mode 100644 index 000000000..56e325221 --- /dev/null +++ b/fixture/ga_start_date_later.json @@ -0,0 +1,38 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": [ + "ga:date" + ], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga:pageviews", + "type": "INTEGER" + } + ] + } + }, + "data": { + "isDataGolden": true, + "rowCount": 849, + "rows": [ + { + "dimensions": [ + "20170118" + ], + "metrics": [ + { + "values": [ + "37" + ] + } + ] + } + ] + }, + "nextPageToken": "1" + } + ] +} \ No newline at end of file diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index f2ed4e929..708850319 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -55,6 +55,14 @@ defmodule Plausible.Google.API do end end + def get_analytics_end_date(access_token, property_or_view) do + if property?(property_or_view) do + Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view) + else + Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view) + end + end + def fetch_verified_properties(auth) do with {:ok, access_token} <- maybe_refresh_token(auth), {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex index 10e29ed84..cbbf84b7f 100644 --- a/lib/plausible/google/ga4/api.ex +++ b/lib/plausible/google/ga4/api.ex @@ -64,6 +64,10 @@ defmodule Plausible.Google.GA4.API do GA4.HTTP.get_analytics_start_date(access_token, property) end + def get_analytics_end_date(access_token, property) do + GA4.HTTP.get_analytics_end_date(access_token, property) + end + def import_analytics(date_range, property, auth, persist_fn) do Logger.debug( "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex index 60c69a527..882be90b5 100644 --- a/lib/plausible/google/ga4/http.ex +++ b/lib/plausible/google/ga4/http.ex @@ -141,7 +141,7 @@ defmodule Plausible.Google.GA4.HTTP do {:error, :authentication_failed} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error}) + Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error}) {:error, :unknown} end end @@ -162,7 +162,7 @@ defmodule Plausible.Google.GA4.HTTP do {:error, :not_found} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error retrieving Google property #{property}", + Sentry.capture_message("Error retrieving GA4 property #{property}", extra: %{error: error} ) @@ -171,7 +171,18 @@ defmodule Plausible.Google.GA4.HTTP do end @earliest_valid_date "2015-08-14" + def get_analytics_start_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :start) + end + + def get_analytics_end_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :end) + end + + defp get_analytics_boundary_date(access_token, property, edge) do + descending? = edge == :end + params = %{ requests: [ %{ @@ -182,7 +193,7 @@ defmodule Plausible.Google.GA4.HTTP do dimensions: [%{name: "date"}], metrics: [%{name: "screenPageViews"}], orderBys: [ - %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false} + %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?} ], limit: 1 } @@ -207,13 +218,15 @@ defmodule Plausible.Google.GA4.HTTP do {:ok, date} - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)}) - {:error, body} + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error retrieving GA4 #{edge} date", + extra: %{error: error} + ) - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)}) - {:error, reason} + {:error, :unknown} end end diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex index 5a38960a5..e93f55faf 100644 --- a/lib/plausible/google/ua/api.ex +++ b/lib/plausible/google/ua/api.ex @@ -64,6 +64,10 @@ defmodule Plausible.Google.UA.API do UA.HTTP.get_analytics_start_date(access_token, view_id) end + def get_analytics_end_date(access_token, view_id) do + UA.HTTP.get_analytics_end_date(access_token, view_id) + end + @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: :ok | {:error, term()} @doc """ diff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.ex index 9d63abfe1..ab04a1f79 100644 --- a/lib/plausible/google/ua/http.ex +++ b/lib/plausible/google/ua/http.ex @@ -109,13 +109,29 @@ defmodule Plausible.Google.UA.HTTP do {:error, :authentication_failed} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) + Sentry.capture_message("Error listing UA views for user", extra: %{error: error}) {:error, :unknown} end end @earliest_valid_date "2005-01-01" + def get_analytics_start_date(access_token, view_id) do + get_analytics_boundary_date(access_token, view_id, :start) + end + + def get_analytics_end_date(access_token, view_id) do + get_analytics_boundary_date(access_token, view_id, :end) + end + + defp get_analytics_boundary_date(access_token, view_id, edge) do + sort_order = + if edge == :start do + "ASCENDING" + else + "DESCENDING" + end + params = %{ reportRequests: [ %{ @@ -127,7 +143,7 @@ defmodule Plausible.Google.UA.HTTP do metrics: [%{expression: "ga:pageviews"}], hideTotals: true, hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], + orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}], pageSize: 1 } ] @@ -151,13 +167,15 @@ defmodule Plausible.Google.UA.HTTP do {:ok, date} - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)}) - {:error, body} + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)}) - {:error, reason} + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error retrieving UA #{edge} date", + extra: %{error: error} + ) + + {:error, :unknown} end end diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex index 18058f6ec..e4a4d7f6c 100644 --- a/lib/plausible/imported.ex +++ b/lib/plausible/imported.ex @@ -66,10 +66,11 @@ defmodule Plausible.Imported do defdelegate listen(), to: Imported.Importer - @spec list_all_imports(Site.t()) :: [SiteImport.t()] - def list_all_imports(site) do + @spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()] + def list_all_imports(site, status \\ nil) do imports = from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at]) + |> maybe_filter_by_status(status) |> Repo.all() if site.imported_data && not Enum.any?(imports, & &1.legacy) do @@ -79,6 +80,12 @@ defmodule Plausible.Imported do end end + defp maybe_filter_by_status(query, nil), do: query + + defp maybe_filter_by_status(query, status) do + where(query, [i], i.status == ^status) + end + @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()] def list_complete_import_ids(site) do ids = @@ -123,4 +130,66 @@ defmodule Plausible.Imported do :ok end + + @spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) :: + {:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window} + def check_dates(_site, nil, _end_date), do: {:error, :no_data} + + def check_dates(site, start_date, end_date) do + cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + end_date = Enum.min([end_date, cutoff_date], Date) + + with true <- Date.diff(end_date, start_date) >= 2, + [_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do + longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first)) + {:ok, longest.first, longest.last} + else + _ -> {:error, :no_time_window} + end + end + + defp find_free_ranges(start_date, end_date, site) do + occupied_ranges = + site + |> Imported.list_all_imports(Imported.SiteImport.completed()) + |> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2)) + |> Enum.map(&Date.range(&1.start_date, &1.end_date)) + + Date.range(start_date, end_date) + |> free_ranges(start_date, occupied_ranges, []) + end + + # This function recursively finds open ranges that are not yet occupied + # by existing imported data. The idea is that we keep moving a dynamic + # date index `d` from start until the end of `imported_range`, hopping + # over each occupied range, and capturing the open ranges step-by-step + # in the `result` array. + defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do + cond do + Date.diff(occupied_range.last, d) <= 0 -> + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 -> + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + true -> + free_range = Date.range(d, occupied_range.first) + result = result ++ [free_range] + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + end + end + + defp free_ranges(import_range, d, [], result) do + if Date.diff(import_range.last, d) < 2 do + result + else + result ++ [Date.range(d, import_range.last)] + end + end + + defp in_range?(date, range) do + Date.before?(range.first, date) && Date.after?(range.last, date) + end end diff --git a/lib/plausible_web/controllers/google_analytics_controller.ex b/lib/plausible_web/controllers/google_analytics_controller.ex index 05cb52736..f700e5f65 100644 --- a/lib/plausible_web/controllers/google_analytics_controller.ex +++ b/lib/plausible_web/controllers/google_analytics_controller.ex @@ -1,6 +1,11 @@ defmodule PlausibleWeb.GoogleAnalyticsController do use PlausibleWeb, :controller + alias Plausible.Google + alias Plausible.Imported + + require Plausible.Imported.SiteImport + plug(PlausibleWeb.RequireAccountPlug) plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) @@ -10,6 +15,8 @@ defmodule PlausibleWeb.GoogleAnalyticsController do "access_token" => access_token, "refresh_token" => refresh_token, "expires_at" => expires_at, + "start_date" => start_date, + "end_date" => end_date, "legacy" => legacy }) do site = conn.assigns.site @@ -22,17 +29,22 @@ defmodule PlausibleWeb.GoogleAnalyticsController do access_token: access_token, refresh_token: refresh_token, expires_at: expires_at, + start_date: start_date, + end_date: end_date, legacy: legacy, layout: {PlausibleWeb.LayoutView, "focus.html"} ) end - def property_or_view_form(conn, %{ - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do + def property_or_view_form( + conn, + %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + } = params + ) do site = conn.assigns.site redirect_route = @@ -44,9 +56,21 @@ defmodule PlausibleWeb.GoogleAnalyticsController do result = if legacy == "true" do - Plausible.Google.UA.API.list_views(access_token) + Google.UA.API.list_views(access_token) else - Plausible.Google.API.list_properties_and_views(access_token) + Google.API.list_properties_and_views(access_token) + end + + error = + case params["error"] do + "no_data" -> + "No data found. Nothing to import." + + "no_time_window" -> + "Imported data time range is completely overlapping with existing data. Nothing to import." + + _ -> + nil end case result do @@ -59,6 +83,7 @@ defmodule PlausibleWeb.GoogleAnalyticsController do expires_at: expires_at, site: conn.assigns.site, properties_and_views: properties_and_views, + selected_property_or_view_error: error, legacy: legacy, layout: {PlausibleWeb.LayoutView, "focus.html"} ) @@ -75,65 +100,81 @@ defmodule PlausibleWeb.GoogleAnalyticsController do conn |> put_flash( :error, - "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + "We were unable to list your Google Analytics properties and views. If the problem persists, please contact support for assistance." ) |> redirect(external: redirect_route) end end # see https://stackoverflow.com/a/57416769 - @google_analytics_new_user_metric_date ~D[2016-08-24] + @universal_analytics_new_user_metric_date ~D[2016-08-24] - def property_or_view(conn, %{ - "property_or_view" => property_or_view, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do + def property_or_view( + conn, + %{ + "property_or_view" => property_or_view, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + } = params + ) do site = conn.assigns.site - start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view) - case start_date do - {:ok, nil} -> - {:ok, properties_and_views} = - if legacy == "true" do - Plausible.Google.UA.API.list_views(access_token) - else - Plausible.Google.API.list_properties_and_views(access_token) - end + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + with {:ok, api_start_date} <- + Google.API.get_analytics_start_date(access_token, property_or_view), + {:ok, api_end_date} <- Google.API.get_analytics_end_date(access_token, property_or_view), + {:ok, start_date, end_date} <- Imported.check_dates(site, api_start_date, api_end_date) do + action = + if Timex.before?(api_start_date, @universal_analytics_new_user_metric_date) do + :user_metric_notice + else + :confirm + end + + redirect(conn, + external: + Routes.google_analytics_path(conn, action, site.domain, + property_or_view: property_or_view, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + start_date: Date.to_iso8601(start_date), + end_date: Date.to_iso8601(end_date), + legacy: legacy + ) + ) + else + {:error, error} when error in [:no_data, :no_time_window] -> + params = + params + |> Map.take(["access_token", "refresh_token", "expires_at", "legacy"]) + |> Map.put("error", Atom.to_string(error)) + + property_or_view_form(conn, params) + {:error, :authentication_failed} -> conn - |> assign(:skip_plausible_tracking, true) - |> render("property_or_view_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - properties_and_views: properties_and_views, - selected_property_or_view_error: "No data found. Nothing to import", - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} + |> put_flash( + :error, + "Google Analytics authentication seems to have expired. Please try again." ) + |> redirect(external: redirect_route) - {:ok, date} -> - action = - if Timex.before?(date, @google_analytics_new_user_metric_date) do - :user_metric_notice - else - :confirm - end - - redirect(conn, - to: - Routes.google_analytics_path(conn, action, site.domain, - property_or_view: property_or_view, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." ) + |> redirect(external: redirect_route) end end @@ -142,32 +183,64 @@ defmodule PlausibleWeb.GoogleAnalyticsController do "access_token" => access_token, "refresh_token" => refresh_token, "expires_at" => expires_at, + "start_date" => start_date, + "end_date" => end_date, "legacy" => legacy }) do site = conn.assigns.site - start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view) + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + case Google.API.get_property_or_view(access_token, property_or_view) do + {:ok, %{name: property_or_view_name, id: property_or_view}} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_property_or_view: property_or_view, + selected_property_or_view_name: property_or_view_name, + start_date: start_date, + end_date: end_date, + property?: Google.API.property?(property_or_view), + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) - end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "Google Analytics authentication seems to have expired. Please try again." + ) + |> redirect(external: redirect_route) - {:ok, %{name: property_or_view_name, id: property_or_view}} = - Plausible.Google.API.get_property_or_view(access_token, property_or_view) + {:error, :not_found} -> + conn + |> put_flash( + :error, + "Google Analytics property not found. Please try again." + ) + |> redirect(external: redirect_route) - conn - |> assign(:skip_plausible_tracking, true) - |> render("confirm.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - selected_property_or_view: property_or_view, - selected_property_or_view_name: property_or_view_name, - start_date: start_date, - end_date: end_date, - property?: Plausible.Google.API.property?(property_or_view), - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end end def import(conn, %{ @@ -182,6 +255,9 @@ defmodule PlausibleWeb.GoogleAnalyticsController do site = conn.assigns.site current_user = conn.assigns.current_user + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + redirect_route = if legacy == "true" do Routes.site_path(conn, :settings_integrations, site.domain) @@ -189,36 +265,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController do Routes.site_path(conn, :settings_imports_exports, site.domain) end - if Plausible.Google.API.property?(property_or_view) do - {:ok, _} = - Plausible.Imported.GoogleAnalytics4.new_import( - site, - current_user, - property: property_or_view, - label: property_or_view, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at + case Imported.check_dates(site, start_date, end_date) do + {:ok, start_date, end_date} -> + if Google.API.property?(property_or_view) do + {:ok, _} = + Imported.GoogleAnalytics4.new_import( + site, + current_user, + property: property_or_view, + label: property_or_view, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at + ) + else + {:ok, _} = + Imported.UniversalAnalytics.new_import( + site, + current_user, + view_id: property_or_view, + label: property_or_view, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at, + legacy: legacy == "true" + ) + end + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + + {:error, :no_time_window} -> + conn + |> put_flash( + :error, + "Import failed. No data could be imported because date range overlaps with existing data." ) - else - Plausible.Imported.UniversalAnalytics.new_import( - site, - current_user, - view_id: property_or_view, - label: property_or_view, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at, - legacy: legacy == "true" - ) + |> redirect(external: redirect_route) end - - conn - |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") - |> redirect(external: redirect_route) end end diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.ex index fc9d170df..dffd6b244 100644 --- a/lib/plausible_web/live/imports_exports_settings.ex +++ b/lib/plausible_web/live/imports_exports_settings.ex @@ -62,9 +62,11 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
- Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %>
+ <%= Plausible.Imported.SiteImport.label(entry.site_import) %>
- (<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views)
+ (<%= PlausibleWeb.StatsView.large_number_format(
+ Map.get(@pageview_counts, entry.site_import.id, 0)
+ ) %> page views)
- The following error occurred when fetching your Google Analytics data.
- <%= error %>