diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 92166cb01..85531658d 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -104,36 +104,55 @@ defmodule Plausible.Google.Api do end @per_page 10_000 + @max_attempts 5 + @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) :: + :ok | {:error, term()} @doc """ - API reference: - https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest + Imports stats from a Google Analytics UA view to a Plausible site. + + This function fetches Google Analytics reports in batches of #{@per_page} per + request. The batches are then buffered to Clickhouse by the + `Plausible.Google.Buffer` process. + + Requests to Google Analytics can fail, and are retried at most + #{@max_attempts} times. Returns `:ok` when importing has finished or + `{:error, term()}` when a request to GA failed too many times. + + Useful links: + + - [Feature documentation](https://plausible.io/docs/google-analytics-import) + - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest) + - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) - Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer """ def import_analytics(site, date_range, view_id, access_token) do {:ok, buffer} = Plausible.Google.Buffer.start_link() - ReportRequest.full_report() - |> Enum.each(fn %ReportRequest{} = report_request -> - report_request = %ReportRequest{ - report_request - | date_range: date_range, - view_id: view_id, - access_token: access_token, - page_token: nil, - page_size: @per_page - } - - fetch_and_persist(site, report_request, buffer: buffer) - end) + result = + Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok -> + report_request = %ReportRequest{ + report_request + | date_range: date_range, + view_id: view_id, + access_token: access_token, + page_token: nil, + page_size: @per_page + } + + case fetch_and_persist(site, report_request, buffer: buffer) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) Plausible.Google.Buffer.flush(buffer) Plausible.Google.Buffer.stop(buffer) - :ok + result end - @max_attempts 5 + @spec fetch_and_persist(Plausible.Site.t(), ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} def fetch_and_persist(site, %ReportRequest{} = report_request, opts \\ []) do buffer_pid = Keyword.get(opts, :buffer) attempt = Keyword.get(opts, :attempt, 1) @@ -155,12 +174,13 @@ defmodule Plausible.Google.Api do :ok end - error -> - context_key = "request:#{attempt}" - Sentry.Context.set_extra_context(%{context_key => error}) - + {:error, cause} -> if attempt >= @max_attempts do - raise "Google API request failed too many times" + Sentry.capture_message("Failed to import from Google Analytics", + extra: %{site: site.domain, error: inspect(cause)} + ) + + {:error, cause} else Process.sleep(sleep_time) fetch_and_persist(site, report_request, Keyword.merge(opts, attempt: attempt + 1)) diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex index a76a53a1c..92ff98313 100644 --- a/lib/plausible/google/http.ex +++ b/lib/plausible/google/http.ex @@ -41,6 +41,13 @@ defmodule Plausible.Google.HTTP do token <- Map.get(report, "nextPageToken"), {:ok, report} <- convert_to_maps(report) do {:ok, {report, token}} + else + {:ok, %{status: _non_http_200, body: body}} -> + Sentry.Context.set_extra_context(%{google_analytics_response: body}) + {:error, :request_failed} + + {:error, cause} -> + {:error, cause} end end diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 63b508a9d..cac81039e 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -15,6 +15,8 @@ defmodule Plausible.Site do alias Plausible.Auth.User alias Plausible.Site.GoogleAuth + @type t() :: %__MODULE__{} + @derive {Jason.Encoder, only: [:domain, :timezone]} schema "sites" do field :domain, :string diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs index 030217fbc..72cb45726 100644 --- a/test/plausible/google/api_test.exs +++ b/test/plausible/google/api_test.exs @@ -70,13 +70,12 @@ defmodule Plausible.Google.ApiTest do page_size: 10_000 } - assert_raise RuntimeError, "Google API request failed too many times", fn -> - Api.fetch_and_persist(site, request, - http_client: finch_double, - sleep_time: 0, - buffer: buffer - ) - end + assert {:error, :request_failed} = + Api.fetch_and_persist(site, request, + http_client: finch_double, + sleep_time: 0, + buffer: buffer + ) assert_receive({Finch, :request, [_, _]}) assert_receive({Finch, :request, [_, _]})