diff --git a/fixture/ga4_end_date.json b/fixture/ga4_end_date.json new file mode 100644 index 000000000..bd066f006 --- /dev/null +++ b/fixture/ga4_end_date.json @@ -0,0 +1,38 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240302" + } + ], + "metricValues": [ + { + "value": "13" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga_end_date.json b/fixture/ga_end_date.json new file mode 100644 index 000000000..68d02b4e7 --- /dev/null +++ b/fixture/ga_end_date.json @@ -0,0 +1,38 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": [ + "ga:date" + ], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga:pageviews", + "type": "INTEGER" + } + ] + } + }, + "data": { + "isDataGolden": true, + "rowCount": 849, + "rows": [ + { + "dimensions": [ + "20200421" + ], + "metrics": [ + { + "values": [ + "37" + ] + } + ] + } + ] + }, + "nextPageToken": "1" + } + ] +} \ No newline at end of file diff --git a/fixture/ga_start_date_later.json b/fixture/ga_start_date_later.json new file mode 100644 index 000000000..56e325221 --- /dev/null +++ b/fixture/ga_start_date_later.json @@ -0,0 +1,38 @@ +{ + "reports": [ + { + "columnHeader": { + "dimensions": [ + "ga:date" + ], + "metricHeader": { + "metricHeaderEntries": [ + { + "name": "ga:pageviews", + "type": "INTEGER" + } + ] + } + }, + "data": { + "isDataGolden": true, + "rowCount": 849, + "rows": [ + { + "dimensions": [ + "20170118" + ], + "metrics": [ + { + "values": [ + "37" + ] + } + ] + } + ] + }, + "nextPageToken": "1" + } + ] +} \ No newline at end of file diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index f2ed4e929..708850319 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -55,6 +55,14 @@ defmodule Plausible.Google.API do end end + def get_analytics_end_date(access_token, property_or_view) do + if property?(property_or_view) do + Plausible.Google.GA4.API.get_analytics_end_date(access_token, property_or_view) + else + Plausible.Google.UA.API.get_analytics_end_date(access_token, property_or_view) + end + end + def fetch_verified_properties(auth) do with {:ok, access_token} <- maybe_refresh_token(auth), {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex index 10e29ed84..cbbf84b7f 100644 --- a/lib/plausible/google/ga4/api.ex +++ b/lib/plausible/google/ga4/api.ex @@ -64,6 +64,10 @@ defmodule Plausible.Google.GA4.API do GA4.HTTP.get_analytics_start_date(access_token, property) end + def get_analytics_end_date(access_token, property) do + GA4.HTTP.get_analytics_end_date(access_token, property) + end + def import_analytics(date_range, property, auth, persist_fn) do Logger.debug( "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex index 60c69a527..882be90b5 100644 --- a/lib/plausible/google/ga4/http.ex +++ b/lib/plausible/google/ga4/http.ex @@ -141,7 +141,7 @@ defmodule Plausible.Google.GA4.HTTP do {:error, :authentication_failed} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error}) + Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error}) {:error, :unknown} end end @@ -162,7 +162,7 @@ defmodule Plausible.Google.GA4.HTTP do {:error, :not_found} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error retrieving Google property #{property}", + Sentry.capture_message("Error retrieving GA4 property #{property}", extra: %{error: error} ) @@ -171,7 +171,18 @@ defmodule Plausible.Google.GA4.HTTP do end @earliest_valid_date "2015-08-14" + def get_analytics_start_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :start) + end + + def get_analytics_end_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :end) + end + + defp get_analytics_boundary_date(access_token, property, edge) do + descending? = edge == :end + params = %{ requests: [ %{ @@ -182,7 +193,7 @@ defmodule Plausible.Google.GA4.HTTP do dimensions: [%{name: "date"}], metrics: [%{name: "screenPageViews"}], orderBys: [ - %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false} + %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?} ], limit: 1 } @@ -207,13 +218,15 @@ defmodule Plausible.Google.GA4.HTTP do {:ok, date} - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)}) - {:error, body} + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error retrieving GA4 #{edge} date", + extra: %{error: error} + ) - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)}) - {:error, reason} + {:error, :unknown} end end diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex index 5a38960a5..e93f55faf 100644 --- a/lib/plausible/google/ua/api.ex +++ b/lib/plausible/google/ua/api.ex @@ -64,6 +64,10 @@ defmodule Plausible.Google.UA.API do UA.HTTP.get_analytics_start_date(access_token, view_id) end + def get_analytics_end_date(access_token, view_id) do + UA.HTTP.get_analytics_end_date(access_token, view_id) + end + @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: :ok | {:error, term()} @doc """ diff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.ex index 9d63abfe1..ab04a1f79 100644 --- a/lib/plausible/google/ua/http.ex +++ b/lib/plausible/google/ua/http.ex @@ -109,13 +109,29 @@ defmodule Plausible.Google.UA.HTTP do {:error, :authentication_failed} {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) + Sentry.capture_message("Error listing UA views for user", extra: %{error: error}) {:error, :unknown} end end @earliest_valid_date "2005-01-01" + def get_analytics_start_date(access_token, view_id) do + get_analytics_boundary_date(access_token, view_id, :start) + end + + def get_analytics_end_date(access_token, view_id) do + get_analytics_boundary_date(access_token, view_id, :end) + end + + defp get_analytics_boundary_date(access_token, view_id, edge) do + sort_order = + if edge == :start do + "ASCENDING" + else + "DESCENDING" + end + params = %{ reportRequests: [ %{ @@ -127,7 +143,7 @@ defmodule Plausible.Google.UA.HTTP do metrics: [%{expression: "ga:pageviews"}], hideTotals: true, hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], + orderBys: [%{fieldName: "ga:date", sortOrder: sort_order}], pageSize: 1 } ] @@ -151,13 +167,15 @@ defmodule Plausible.Google.UA.HTTP do {:ok, date} - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)}) - {:error, body} + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)}) - {:error, reason} + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error retrieving UA #{edge} date", + extra: %{error: error} + ) + + {:error, :unknown} end end diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex index 18058f6ec..e4a4d7f6c 100644 --- a/lib/plausible/imported.ex +++ b/lib/plausible/imported.ex @@ -66,10 +66,11 @@ defmodule Plausible.Imported do defdelegate listen(), to: Imported.Importer - @spec list_all_imports(Site.t()) :: [SiteImport.t()] - def list_all_imports(site) do + @spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()] + def list_all_imports(site, status \\ nil) do imports = from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at]) + |> maybe_filter_by_status(status) |> Repo.all() if site.imported_data && not Enum.any?(imports, & &1.legacy) do @@ -79,6 +80,12 @@ defmodule Plausible.Imported do end end + defp maybe_filter_by_status(query, nil), do: query + + defp maybe_filter_by_status(query, status) do + where(query, [i], i.status == ^status) + end + @spec list_complete_import_ids(Site.t()) :: [non_neg_integer()] def list_complete_import_ids(site) do ids = @@ -123,4 +130,66 @@ defmodule Plausible.Imported do :ok end + + @spec check_dates(Site.t(), Date.t() | nil, Date.t() | nil) :: + {:ok, Date.t(), Date.t()} | {:error, :no_data | :no_time_window} + def check_dates(_site, nil, _end_date), do: {:error, :no_data} + + def check_dates(site, start_date, end_date) do + cutoff_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + end_date = Enum.min([end_date, cutoff_date], Date) + + with true <- Date.diff(end_date, start_date) >= 2, + [_ | _] = free_ranges <- find_free_ranges(start_date, end_date, site) do + longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first)) + {:ok, longest.first, longest.last} + else + _ -> {:error, :no_time_window} + end + end + + defp find_free_ranges(start_date, end_date, site) do + occupied_ranges = + site + |> Imported.list_all_imports(Imported.SiteImport.completed()) + |> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2)) + |> Enum.map(&Date.range(&1.start_date, &1.end_date)) + + Date.range(start_date, end_date) + |> free_ranges(start_date, occupied_ranges, []) + end + + # This function recursively finds open ranges that are not yet occupied + # by existing imported data. The idea is that we keep moving a dynamic + # date index `d` from start until the end of `imported_range`, hopping + # over each occupied range, and capturing the open ranges step-by-step + # in the `result` array. + defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do + cond do + Date.diff(occupied_range.last, d) <= 0 -> + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 -> + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + true -> + free_range = Date.range(d, occupied_range.first) + result = result ++ [free_range] + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + end + end + + defp free_ranges(import_range, d, [], result) do + if Date.diff(import_range.last, d) < 2 do + result + else + result ++ [Date.range(d, import_range.last)] + end + end + + defp in_range?(date, range) do + Date.before?(range.first, date) && Date.after?(range.last, date) + end end diff --git a/lib/plausible_web/controllers/google_analytics_controller.ex b/lib/plausible_web/controllers/google_analytics_controller.ex index 05cb52736..f700e5f65 100644 --- a/lib/plausible_web/controllers/google_analytics_controller.ex +++ b/lib/plausible_web/controllers/google_analytics_controller.ex @@ -1,6 +1,11 @@ defmodule PlausibleWeb.GoogleAnalyticsController do use PlausibleWeb, :controller + alias Plausible.Google + alias Plausible.Imported + + require Plausible.Imported.SiteImport + plug(PlausibleWeb.RequireAccountPlug) plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) @@ -10,6 +15,8 @@ defmodule PlausibleWeb.GoogleAnalyticsController do "access_token" => access_token, "refresh_token" => refresh_token, "expires_at" => expires_at, + "start_date" => start_date, + "end_date" => end_date, "legacy" => legacy }) do site = conn.assigns.site @@ -22,17 +29,22 @@ defmodule PlausibleWeb.GoogleAnalyticsController do access_token: access_token, refresh_token: refresh_token, expires_at: expires_at, + start_date: start_date, + end_date: end_date, legacy: legacy, layout: {PlausibleWeb.LayoutView, "focus.html"} ) end - def property_or_view_form(conn, %{ - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do + def property_or_view_form( + conn, + %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + } = params + ) do site = conn.assigns.site redirect_route = @@ -44,9 +56,21 @@ defmodule PlausibleWeb.GoogleAnalyticsController do result = if legacy == "true" do - Plausible.Google.UA.API.list_views(access_token) + Google.UA.API.list_views(access_token) else - Plausible.Google.API.list_properties_and_views(access_token) + Google.API.list_properties_and_views(access_token) + end + + error = + case params["error"] do + "no_data" -> + "No data found. Nothing to import." + + "no_time_window" -> + "Imported data time range is completely overlapping with existing data. Nothing to import." + + _ -> + nil end case result do @@ -59,6 +83,7 @@ defmodule PlausibleWeb.GoogleAnalyticsController do expires_at: expires_at, site: conn.assigns.site, properties_and_views: properties_and_views, + selected_property_or_view_error: error, legacy: legacy, layout: {PlausibleWeb.LayoutView, "focus.html"} ) @@ -75,65 +100,81 @@ defmodule PlausibleWeb.GoogleAnalyticsController do conn |> put_flash( :error, - "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + "We were unable to list your Google Analytics properties and views. If the problem persists, please contact support for assistance." ) |> redirect(external: redirect_route) end end # see https://stackoverflow.com/a/57416769 - @google_analytics_new_user_metric_date ~D[2016-08-24] + @universal_analytics_new_user_metric_date ~D[2016-08-24] - def property_or_view(conn, %{ - "property_or_view" => property_or_view, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do + def property_or_view( + conn, + %{ + "property_or_view" => property_or_view, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + } = params + ) do site = conn.assigns.site - start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view) - case start_date do - {:ok, nil} -> - {:ok, properties_and_views} = - if legacy == "true" do - Plausible.Google.UA.API.list_views(access_token) - else - Plausible.Google.API.list_properties_and_views(access_token) - end + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + with {:ok, api_start_date} <- + Google.API.get_analytics_start_date(access_token, property_or_view), + {:ok, api_end_date} <- Google.API.get_analytics_end_date(access_token, property_or_view), + {:ok, start_date, end_date} <- Imported.check_dates(site, api_start_date, api_end_date) do + action = + if Timex.before?(api_start_date, @universal_analytics_new_user_metric_date) do + :user_metric_notice + else + :confirm + end + + redirect(conn, + external: + Routes.google_analytics_path(conn, action, site.domain, + property_or_view: property_or_view, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + start_date: Date.to_iso8601(start_date), + end_date: Date.to_iso8601(end_date), + legacy: legacy + ) + ) + else + {:error, error} when error in [:no_data, :no_time_window] -> + params = + params + |> Map.take(["access_token", "refresh_token", "expires_at", "legacy"]) + |> Map.put("error", Atom.to_string(error)) + + property_or_view_form(conn, params) + {:error, :authentication_failed} -> conn - |> assign(:skip_plausible_tracking, true) - |> render("property_or_view_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - properties_and_views: properties_and_views, - selected_property_or_view_error: "No data found. Nothing to import", - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} + |> put_flash( + :error, + "Google Analytics authentication seems to have expired. Please try again." ) + |> redirect(external: redirect_route) - {:ok, date} -> - action = - if Timex.before?(date, @google_analytics_new_user_metric_date) do - :user_metric_notice - else - :confirm - end - - redirect(conn, - to: - Routes.google_analytics_path(conn, action, site.domain, - property_or_view: property_or_view, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." ) + |> redirect(external: redirect_route) end end @@ -142,32 +183,64 @@ defmodule PlausibleWeb.GoogleAnalyticsController do "access_token" => access_token, "refresh_token" => refresh_token, "expires_at" => expires_at, + "start_date" => start_date, + "end_date" => end_date, "legacy" => legacy }) do site = conn.assigns.site - start_date = Plausible.Google.API.get_analytics_start_date(access_token, property_or_view) + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + case Google.API.get_property_or_view(access_token, property_or_view) do + {:ok, %{name: property_or_view_name, id: property_or_view}} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_property_or_view: property_or_view, + selected_property_or_view_name: property_or_view_name, + start_date: start_date, + end_date: end_date, + property?: Google.API.property?(property_or_view), + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) - end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "Google Analytics authentication seems to have expired. Please try again." + ) + |> redirect(external: redirect_route) - {:ok, %{name: property_or_view_name, id: property_or_view}} = - Plausible.Google.API.get_property_or_view(access_token, property_or_view) + {:error, :not_found} -> + conn + |> put_flash( + :error, + "Google Analytics property not found. Please try again." + ) + |> redirect(external: redirect_route) - conn - |> assign(:skip_plausible_tracking, true) - |> render("confirm.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - selected_property_or_view: property_or_view, - selected_property_or_view_name: property_or_view_name, - start_date: start_date, - end_date: end_date, - property?: Plausible.Google.API.property?(property_or_view), - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end end def import(conn, %{ @@ -182,6 +255,9 @@ defmodule PlausibleWeb.GoogleAnalyticsController do site = conn.assigns.site current_user = conn.assigns.current_user + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + redirect_route = if legacy == "true" do Routes.site_path(conn, :settings_integrations, site.domain) @@ -189,36 +265,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController do Routes.site_path(conn, :settings_imports_exports, site.domain) end - if Plausible.Google.API.property?(property_or_view) do - {:ok, _} = - Plausible.Imported.GoogleAnalytics4.new_import( - site, - current_user, - property: property_or_view, - label: property_or_view, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at + case Imported.check_dates(site, start_date, end_date) do + {:ok, start_date, end_date} -> + if Google.API.property?(property_or_view) do + {:ok, _} = + Imported.GoogleAnalytics4.new_import( + site, + current_user, + property: property_or_view, + label: property_or_view, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at + ) + else + {:ok, _} = + Imported.UniversalAnalytics.new_import( + site, + current_user, + view_id: property_or_view, + label: property_or_view, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at, + legacy: legacy == "true" + ) + end + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + + {:error, :no_time_window} -> + conn + |> put_flash( + :error, + "Import failed. No data could be imported because date range overlaps with existing data." ) - else - Plausible.Imported.UniversalAnalytics.new_import( - site, - current_user, - view_id: property_or_view, - label: property_or_view, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at, - legacy: legacy == "true" - ) + |> redirect(external: redirect_route) end - - conn - |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") - |> redirect(external: redirect_route) end end diff --git a/lib/plausible_web/live/imports_exports_settings.ex b/lib/plausible_web/live/imports_exports_settings.ex index fc9d170df..dffd6b244 100644 --- a/lib/plausible_web/live/imports_exports_settings.ex +++ b/lib/plausible_web/live/imports_exports_settings.ex @@ -62,9 +62,11 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
  • - Import from <%= Plausible.Imported.SiteImport.label(entry.site_import) %> + <%= Plausible.Imported.SiteImport.label(entry.site_import) %> - (<%= Map.get(@pageview_counts, entry.site_import.id, 0) %> page views) + (<%= PlausibleWeb.StatsView.large_number_format( + Map.get(@pageview_counts, entry.site_import.id, 0) + ) %> page views) <%= hidden_input(f, :legacy, value: @legacy) %> - <%= case @start_date do %> - <% {:ok, start_date} -> %> -

    - Stats from this - <%= if @property? do %> - property - <% else %> - view - <% end %> - and time period will be imported from your Google Analytics account to your Plausible dashboard -
    +
    + Stats from this + <%= if @property? do %> + property + <% else %> + view + <% end %> + and time period will be imported from your Google Analytics account to your Plausible dashboard +
    -
    - <%= styled_label( - f, - :property_or_view, - "Google Analytics #{if @property?, do: "property", else: "view"}" - ) %> - - <%= @selected_property_or_view_name %> - - <%= hidden_input(f, :property_or_view, - readonly: "true", - value: @selected_property_or_view - ) %> -
    -
    -
    - <%= styled_label(f, :start_date, "From") %> - - <%= PlausibleWeb.EmailView.date_format(start_date) %> - - <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %> -
    -
    →
    -
    - <%= styled_label(f, :end_date, "To") %> - - <%= PlausibleWeb.EmailView.date_format(@end_date) %> - - <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %> -
    -
    - <% {:error, error} -> %> -

    - The following error occurred when fetching your Google Analytics data. -

    -

    <%= error %>

    - <% end %> +
    + <%= styled_label( + f, + :property_or_view, + "Google Analytics #{if @property?, do: "property", else: "view"}" + ) %> + + <%= @selected_property_or_view_name %> + + <%= hidden_input(f, :property_or_view, + readonly: "true", + value: @selected_property_or_view + ) %> +
    +
    +
    + <%= styled_label(f, :start_date, "From") %> + + <%= PlausibleWeb.EmailView.date_format(@start_date) %> + + <%= hidden_input(f, :start_date, value: @start_date, readonly: "true") %> +
    +
    →
    +
    + <%= styled_label(f, :end_date, "To") %> + + <%= PlausibleWeb.EmailView.date_format(@end_date) %> + + <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %> +
    +
    <%= submit("Confirm import", class: "button mt-6") %> <% end %> diff --git a/lib/plausible_web/templates/google_analytics/user_metric_form.html.heex b/lib/plausible_web/templates/google_analytics/user_metric_form.html.heex index f6f3adf0b..e17a3ee2a 100644 --- a/lib/plausible_web/templates/google_analytics/user_metric_form.html.heex +++ b/lib/plausible_web/templates/google_analytics/user_metric_form.html.heex @@ -39,6 +39,8 @@ access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, + start_date: @start_date, + end_date: @end_date, legacy: @legacy ), class: "button mt-6" diff --git a/test/plausible/google/ga4/api_test.exs b/test/plausible/google/ga4/api_test.exs index ed838c8af..98fbdcc34 100644 --- a/test/plausible/google/ga4/api_test.exs +++ b/test/plausible/google/ga4/api_test.exs @@ -55,4 +55,17 @@ defmodule Plausible.Google.GA4.APITest do GA4.API.get_analytics_start_date("some_access_token", "properties/153293282") end end + + describe "get_analytics_end_date/2" do + test "returns stats end date for a given property" do + result = Jason.decode!(File.read!("fixture/ga4_end_date.json")) + + expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body -> + {:ok, %Finch.Response{status: 200, body: result}} + end) + + assert {:ok, ~D[2024-03-02]} = + GA4.API.get_analytics_end_date("some_access_token", "properties/153293282") + end + end end diff --git a/test/plausible/imported_test.exs b/test/plausible/imported_test.exs index ee98f5c37..e5a6d0ba1 100644 --- a/test/plausible/imported_test.exs +++ b/test/plausible/imported_test.exs @@ -59,4 +59,149 @@ defmodule Plausible.ImportedTest do ] = Imported.list_all_imports(site) end end + + describe "check_dates/3" do + test "crops dates from both ends when overlapping with existing import and native stats" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + start_date = ~D[2016-04-03] + end_date = ~D[2021-05-12] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + + assert {:ok, ~D[2021-05-12], ~D[2023-10-25]} = + Imported.check_dates(site, ~D[2021-04-11], ~D[2024-01-12]) + end + + test "picks longest continuous range when containing existing import" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + start_date = ~D[2019-04-03] + end_date = ~D[2021-05-12] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + + assert {:ok, ~D[2021-05-12], ~D[2023-10-25]} = + Imported.check_dates(site, ~D[2019-03-21], ~D[2024-01-12]) + end + + test "does not alter the dates when there are no imports and no native stats" do + site = insert(:site) + + assert {:ok, ~D[2021-05-12], ~D[2024-01-12]} = + Imported.check_dates(site, ~D[2021-05-12], ~D[2024-01-12]) + end + + test "ignores input date range difference smaller than 2 days" do + site = insert(:site) + + assert {:error, :no_time_window} = + Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-12]) + + assert {:error, :no_time_window} = + Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-13]) + + assert {:ok, ~D[2024-01-12], ~D[2024-01-14]} = + Imported.check_dates(site, ~D[2024-01-12], ~D[2024-01-14]) + end + + test "ignores imports with date range difference smaller than 2 days" do + site = insert(:site) + + start_date = ~D[2024-01-12] + end_date = ~D[2024-01-13] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + + assert {:ok, ~D[2021-04-22], ~D[2024-03-14]} = + Imported.check_dates(site, ~D[2021-04-22], ~D[2024-03-14]) + end + + test "returns no time window when input range starts after native stats start date" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + assert {:error, :no_time_window} = + Imported.check_dates(site, ~D[2023-10-28], ~D[2024-01-13]) + end + + test "returns no time window when input range starts less than 2 days before native stats start date" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + assert {:error, :no_time_window} = + Imported.check_dates(site, ~D[2023-10-24], ~D[2024-01-13]) + end + + test "crops time range at native stats start date when effective range is 2 days or longer" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + assert {:ok, ~D[2023-10-23], ~D[2023-10-25]} = + Imported.check_dates(site, ~D[2023-10-23], ~D[2024-01-13]) + end + + test "returns no data error when start date missing" do + site = insert(:site) + + assert {:error, :no_data} = Imported.check_dates(site, nil, nil) + end + + test "returns no time window error when date range overlaps with existing import and stats completely" do + site = insert(:site) + + populate_stats(site, [ + build(:pageview, timestamp: ~N[2023-10-25 15:58:00]) + ]) + + start_date = ~D[2016-04-03] + end_date = ~D[2023-10-25] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + + assert {:error, :no_time_window} = + Imported.check_dates(site, ~D[2021-04-11], ~D[2024-01-12]) + end + end end diff --git a/test/plausible_web/controllers/google_analytics_controller_test.exs b/test/plausible_web/controllers/google_analytics_controller_test.exs index 2ffdc7643..10acc30de 100644 --- a/test/plausible_web/controllers/google_analytics_controller_test.exs +++ b/test/plausible_web/controllers/google_analytics_controller_test.exs @@ -5,6 +5,7 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do import Mox import Plausible.Test.Support.HTML + alias Plausible.HTTPClient alias Plausible.Imported.SiteImport require Plausible.Imported.SiteImport @@ -22,6 +23,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do "access_token" => "token", "refresh_token" => "foo", "expires_at" => "2022-09-22T20:01:37.112777", + "start_date" => "2020-02-22", + "end_date" => "2022-09-22", "legacy" => "true" }) |> html_response(200) @@ -32,6 +35,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do access_token: "token", refresh_token: "foo", expires_at: "2022-09-22T20:01:37.112777", + start_date: "2020-02-22", + end_date: "2022-09-22", legacy: "true" ) |> String.replace("&", "&") @@ -100,21 +105,368 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do assert response =~ "GA4 - Flood-It! (properties/153293282)" assert response =~ "GA4 - Google Merch Shop (properties/213025502)" end + + for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do + test "redirects to #{view} on auth error with flash error (legacy: #{legacy})", %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + {:error, %HTTPClient.Non200Error{reason: %{status: 403, body: %{}}}} + end + ) + + conn = + conn + |> get("/#{site.domain}/import/google-analytics/property-or-view", %{ + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "We were unable to authenticate your Google Analytics account" + end + + test "redirects to #{view} on list retrival failure with flash error (legacy: #{legacy})", + %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + {:error, + %HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}} + end + ) + + conn = + conn + |> get("/#{site.domain}/import/google-analytics/property-or-view", %{ + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "We were unable to list your Google Analytics properties and views" + end + end end - describe "GET /:website/import/google-analytics/confirm" do + describe "POST /:website/import/google-analytics/property-or-view" do setup [:create_user, :log_in, :create_new_site] - test "renders confirmation form for Universal Analytics import", %{conn: conn, site: site} do + for legacy <- ["true", "false"] do + test "redirects to user metrics notice (UA legacy: #{legacy})", %{conn: conn, site: site} do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga_start_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga_end_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + conn = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "57238190", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) =~ + "/#{URI.encode_www_form(site.domain)}/import/google-analytics/user-metric" + end + + test "redirects to confirmation (UA legacy: #{legacy})", %{conn: conn, site: site} do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga_start_date_later.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga_end_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + conn = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "57238190", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) =~ + "/#{URI.encode_www_form(site.domain)}/import/google-analytics/confirm" + end + end + + test "redirects to confirmation (GA4)", %{conn: conn, site: site} do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga4_end_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + conn = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => "false" + }) + + assert redirected_to(conn, 302) =~ + "/#{URI.encode_www_form(site.domain)}/import/google-analytics/confirm" + end + + test "renders error when no time window to import available", %{conn: conn, site: site} do + start_date = ~D[2022-01-12] + end_date = ~D[2024-03-13] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + expect( Plausible.HTTPClient.Mock, :post, - fn _url, _headers, _params -> - body = "fixture/ga_start_date.json" |> File.read!() |> Jason.decode!() + fn _url, _opts, _params -> + body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + body = "fixture/ga4_end_date.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + body = "fixture/ga4_list_properties.json" |> File.read!() |> Jason.decode!() {:ok, %Finch.Response{body: body, status: 200}} end ) + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + body = "fixture/ga_list_views.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + response = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => "false" + }) + |> html_response(200) + + assert response =~ + "Imported data time range is completely overlapping with existing data. Nothing to import." + end + + test "renders error when there's no data to import", %{conn: conn, site: site} do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + {:ok, %Finch.Response{body: %{"reports" => []}, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + {:ok, %Finch.Response{body: %{"reports" => []}, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + body = "fixture/ga4_list_properties.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _opts -> + body = "fixture/ga_list_views.json" |> File.read!() |> Jason.decode!() + {:ok, %Finch.Response{body: body, status: 200}} + end + ) + + response = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => "false" + }) + |> html_response(200) + + assert response =~ "No data found. Nothing to import." + end + + for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do + test "redirects to #{view} on failed property/view choice with flash error (legacy: #{legacy})", + %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + {:error, + %HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}} + end + ) + + conn = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "We were unable to retrieve information from Google Analytics" + end + + test "redirects to #{view} on expired authentication with flash error (legacy: #{legacy})", + %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :post, + fn _url, _opts, _params -> + {:error, %HTTPClient.Non200Error{reason: %{status: 403, body: "Access denied"}}} + end + ) + + conn = + conn + |> post("/#{site.domain}/import/google-analytics/property-or-view", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "Google Analytics authentication seems to have expired." + end + end + end + + describe "GET /:website/import/google-analytics/confirm" do + setup [:create_user, :log_in, :create_new_site] + + test "renders confirmation form for Universal Analytics import", %{conn: conn, site: site} do expect( Plausible.HTTPClient.Mock, :get, @@ -131,6 +483,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do "access_token" => "token", "refresh_token" => "foo", "expires_at" => "2022-09-22T20:01:37.112777", + "start_date" => "2012-01-18", + "end_date" => "2022-09-22", "legacy" => "true" }) |> html_response(200) @@ -151,20 +505,10 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do assert text_of_attr(response, ~s|input[name=start_date]|, "value") == "2012-01-18" - assert text_of_attr(response, ~s|input[name=end_date]|, "value") == - Date.to_iso8601(Date.utc_today()) + assert text_of_attr(response, ~s|input[name=end_date]|, "value") == "2022-09-22" end test "renders confirmation form for Google Analytics 4 import", %{conn: conn, site: site} do - expect( - Plausible.HTTPClient.Mock, - :post, - fn _url, _headers, _params -> - body = "fixture/ga4_start_date.json" |> File.read!() |> Jason.decode!() - {:ok, %Finch.Response{body: body, status: 200}} - end - ) - expect( Plausible.HTTPClient.Mock, :get, @@ -181,6 +525,8 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do "access_token" => "token", "refresh_token" => "foo", "expires_at" => "2022-09-22T20:01:37.112777", + "start_date" => "2024-02-22", + "end_date" => "2024-02-26", "legacy" => "true" }) |> html_response(200) @@ -202,8 +548,82 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do assert text_of_attr(response, ~s|input[name=start_date]|, "value") == "2024-02-22" - assert text_of_attr(response, ~s|input[name=end_date]|, "value") == - Date.to_iso8601(Date.utc_today()) + assert text_of_attr(response, ~s|input[name=end_date]|, "value") == "2024-02-26" + end + + for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do + test "redirects to #{view} on failed property/view retrieval with flash error (legacy: #{legacy})", + %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _params -> + {:error, + %HTTPClient.Non200Error{reason: %{status: 500, body: "Internal server error"}}} + end + ) + + conn = + conn + |> get("/#{site.domain}/import/google-analytics/confirm", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "start_date" => "2024-02-22", + "end_date" => "2024-02-26", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "We were unable to retrieve information from Google Analytics" + end + + test "redirects to #{view} on expired authentication with flash error (legacy: #{legacy})", + %{ + conn: conn, + site: site + } do + expect( + Plausible.HTTPClient.Mock, + :get, + fn _url, _params -> + {:error, %HTTPClient.Non200Error{reason: %{status: 403, body: "Access denied"}}} + end + ) + + conn = + conn + |> get("/#{site.domain}/import/google-analytics/confirm", %{ + "property_or_view" => "properties/428685906", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "start_date" => "2024-02-22", + "end_date" => "2024-02-26", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "Google Analytics authentication seems to have expired." + end end end @@ -332,5 +752,44 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do } ) end + + for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do + test "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{ + conn: conn, + site: site + } do + start_date = ~D[2022-01-12] + end_date = ~D[2024-03-13] + + _existing_import = + insert(:site_import, + site: site, + start_date: start_date, + end_date: end_date, + status: :completed + ) + + conn = + post(conn, "/#{site.domain}/settings/google-import", %{ + "property_or_view" => "123456", + "start_date" => "2023-03-01", + "end_date" => "2022-03-01", + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777", + "legacy" => unquote(legacy) + }) + + assert redirected_to(conn, 302) == + PlausibleWeb.Router.Helpers.site_path( + conn, + unquote(view), + site.domain + ) + + assert Phoenix.Flash.get(conn.assigns.flash, :error) =~ + "Import failed. No data could be imported because date range overlaps with existing data." + end + end end end