diff --git a/config/config.exs b/config/config.exs index b3a1f05e8..f7f2f32d8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -45,7 +45,7 @@ config :ref_inspector, config :plausible, paddle_api: Plausible.Billing.PaddleApi, - google_api: Plausible.Google.Api + google_api: Plausible.Google.API config :plausible, # 30 minutes diff --git a/config/test.exs b/config/test.exs index 31182b284..d1d9edfac 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter config :plausible, paddle_api: Plausible.PaddleApi.Mock, - google_api: Plausible.Google.Api.Mock + google_api: Plausible.Google.API.Mock config :bamboo, :refute_timeout, 10 diff --git a/fixture/ga4_list_properties.json b/fixture/ga4_list_properties.json new file mode 100644 index 000000000..f8aac0b1d --- /dev/null +++ b/fixture/ga4_list_properties.json @@ -0,0 +1,41 @@ +{ + "accountSummaries": [ + { + "account": "accounts/28425178", + "displayName": "account.one", + "name": "accountSummaries/28425178", + "propertySummaries": [ + { + "displayName": "account.one - GA4", + "parent": "accounts/28425178", + "property": "properties/428685906", + "propertyType": "PROPERTY_TYPE_ORDINARY" + } + ] + }, + { + "account": "accounts/45336102", + "displayName": "account.two", + "name": "accountSummaries/45336102" + }, + { + "account": "accounts/54516992", + "displayName": "Demo Account", + "name": "accountSummaries/54516992", + "propertySummaries": [ + { + "displayName": "GA4 - Flood-It!", + "parent": "accounts/54516992", + "property": "properties/153293282", + "propertyType": "PROPERTY_TYPE_ORDINARY" + }, + { + "displayName": "GA4 - Google Merch Shop", + "parent": "accounts/54516992", + "property": "properties/213025502", + "propertyType": "PROPERTY_TYPE_ORDINARY" + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_browsers.json b/fixture/ga4_report_imported_browsers.json new file mode 100644 index 000000000..214ac2b26 --- /dev/null +++ b/fixture/ga4_report_imported_browsers.json @@ -0,0 +1,161 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "browser" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 5, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "Safari" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "Chrome" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Chrome" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Firefox" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "2" + }, + { + "value": "0" + }, + { + "value": "21" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Safari" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_devices.json b/fixture/ga4_report_imported_devices.json new file mode 100644 index 000000000..e8bcc0104 --- /dev/null +++ b/fixture/ga4_report_imported_devices.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "deviceCategory" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "mobile" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "desktop" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "desktop" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "mobile" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_entry_pages.json b/fixture/ga4_report_imported_entry_pages.json new file mode 100644 index 000000000..77370bb30 --- /dev/null +++ b/fixture/ga4_report_imported_entry_pages.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "landingPage" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "/blog/firstpost" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + }, + { + "value": "2" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "25" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "/blog/unicode-in-elixir" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "4" + }, + { + "value": "0" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_locations.json b/fixture/ga4_report_imported_locations.json new file mode 100644 index 000000000..b731a2102 --- /dev/null +++ b/fixture/ga4_report_imported_locations.json @@ -0,0 +1,167 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "countryId" + }, + { + "name": "region" + }, + { + "name": "city" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "PL" + }, + { + "value": "Masovian Voivodeship" + }, + { + "value": "Warsaw" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "US" + }, + { + "value": "California" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "PL" + }, + { + "value": "Pomeranian Voivodeship" + }, + { + "value": "Gdansk" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "ES" + }, + { + "value": "Catalonia" + }, + { + "value": "Barcelona" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_operating_systems.json b/fixture/ga4_report_imported_operating_systems.json new file mode 100644 index 000000000..158fe1efa --- /dev/null +++ b/fixture/ga4_report_imported_operating_systems.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "operatingSystem" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "iOS" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "Windows" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Macintosh" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "iOS" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_pages.json b/fixture/ga4_report_imported_pages.json new file mode 100644 index 000000000..f7be04f26 --- /dev/null +++ b/fixture/ga4_report_imported_pages.json @@ -0,0 +1,232 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "hostName" + }, + { + "name": "pagePath" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 8, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/firstpost/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "drawer-4l3.pages.dev" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "7" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/unicode-in-elixir/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "7" + }, + { + "value": "21" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/about/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/firstpost/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_sources.json b/fixture/ga4_report_imported_sources.json new file mode 100644 index 000000000..4c5edb517 --- /dev/null +++ b/fixture/ga4_report_imported_sources.json @@ -0,0 +1,161 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "sessionSource" + }, + { + "name": "sessionMedium" + }, + { + "name": "sessionCampaignName" + }, + { + "name": "sessionManualAdContent" + }, + { + "name": "sessionGoogleAdsKeyword" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "3" + }, + { + "value": "4" + }, + { + "value": "0" + }, + { + "value": "29" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_visitors.json b/fixture/ga4_report_imported_visitors.json new file mode 100644 index 000000000..78d29b5e7 --- /dev/null +++ b/fixture/ga4_report_imported_visitors.json @@ -0,0 +1,114 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + } + ], + "metricValues": [ + { + "value": "3" + }, + { + "value": "13" + }, + { + "value": "0" + }, + { + "value": "4" + }, + { + "value": "29" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_start_date.json b/fixture/ga4_start_date.json new file mode 100644 index 000000000..6c1d410d2 --- /dev/null +++ b/fixture/ga4_start_date.json @@ -0,0 +1,38 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240222" + } + ], + "metricValues": [ + { + "value": "13" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 085513886..6b1e44b4e 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -1,9 +1,13 @@ -defmodule Plausible.Google.Api do - alias Plausible.Google.{ReportRequest, HTTP} +defmodule Plausible.Google.API do + @moduledoc """ + API to Google services. + """ + use Timex - require Logger - @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()} + alias Plausible.Google.HTTP + + require Logger @search_console_scope URI.encode_www_form( "email https://www.googleapis.com/auth/webmasters.readonly" @@ -17,9 +21,16 @@ defmodule Plausible.Google.Api do Jason.encode!([site_id, redirect_to]) end - def import_authorize_url(site_id, redirect_to, legacy \\ true) do + def import_authorize_url(site_id, redirect_to, opts \\ []) do + legacy = Keyword.get(opts, :legacy, true) + ga4 = Keyword.get(opts, :ga4, false) + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> - Jason.encode!([site_id, redirect_to, legacy]) + Jason.encode!([site_id, redirect_to, legacy, ga4]) + end + + def fetch_access_token!(code) do + HTTP.fetch_access_token!(code) end def fetch_verified_properties(auth) do @@ -53,138 +64,7 @@ defmodule Plausible.Google.Api do end end - @spec list_views(access_token :: String.t()) :: - {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()} - @doc """ - Lists Google Analytics views grouped by hostname. - """ - def list_views(access_token) do - case HTTP.list_views_for_user(access_token) do - {:ok, %{"items" => views}} -> - views = Enum.group_by(views, &view_hostname/1, &view_names/1) - {:ok, views} - - error -> - error - end - end - - defp view_hostname(view) do - case view do - %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host) - _any -> "Others" - end - end - - defp view_names(%{"name" => name, "id" => id}) do - {"#{id} - #{name}", id} - end - - @spec get_view(access_token :: String.t(), lookup_id :: String.t()) :: - {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()} - @doc """ - Returns a single Google Analytics view if the user has access to it. - """ - def get_view(access_token, lookup_id) do - case list_views(access_token) do - {:ok, views} -> - view = - views - |> Map.values() - |> List.flatten() - |> Enum.find(fn {_name, id} -> id == lookup_id end) - - {:ok, view} - - {:error, cause} -> - {:error, cause} - end - end - - @type import_auth :: { - access_token :: String.t(), - refresh_token :: String.t(), - expires_at :: String.t() - } - - @per_page 7_500 - @backoff_factor :timer.seconds(10) - @max_attempts 5 - @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: - :ok | {:error, term()} - @doc """ - Imports stats from a Google Analytics UA view to a Plausible site. - - This function fetches Google Analytics reports in batches of #{@per_page} per - request. The batches are then passed to persist callback. - - Requests to Google Analytics can fail, and are retried at most - #{@max_attempts} times with an exponential backoff. Returns `:ok` when - importing has finished or `{:error, term()}` when a request to GA failed too - many times. - - Useful links: - - - [Feature documentation](https://plausible.io/docs/google-analytics-import) - - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest) - - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) - - """ - def import_analytics(date_range, view_id, auth, persist_fn) do - with {:ok, access_token} <- maybe_refresh_token(auth) do - do_import_analytics(date_range, view_id, access_token, persist_fn) - end - end - - defp do_import_analytics(date_range, view_id, access_token, persist_fn) do - Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok -> - report_request = %ReportRequest{ - report_request - | date_range: date_range, - view_id: view_id, - access_token: access_token, - page_token: nil, - page_size: @per_page - } - - case fetch_and_persist(report_request, persist_fn: persist_fn) do - :ok -> {:cont, :ok} - {:error, _} = error -> {:halt, error} - end - end) - end - - @spec fetch_and_persist(ReportRequest.t(), Keyword.t()) :: - :ok | {:error, term()} - def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do - persist_fn = Keyword.fetch!(opts, :persist_fn) - attempt = Keyword.get(opts, :attempt, 1) - sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) - - case HTTP.get_report(report_request) do - {:ok, {rows, next_page_token}} -> - :ok = persist_fn.(report_request.dataset, rows) - - if next_page_token do - fetch_and_persist( - %ReportRequest{report_request | page_token: next_page_token}, - opts - ) - else - :ok - end - - {:error, cause} -> - if attempt >= @max_attempts do - {:error, cause} - else - Process.sleep(attempt * sleep_time) - fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) - end - end - end - - defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do + def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do with true <- needs_to_refresh_token?(auth.expires), {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), changeset <- @@ -200,7 +80,7 @@ defmodule Plausible.Google.Api do end end - defp maybe_refresh_token({access_token, refresh_token, expires_at}) do + def maybe_refresh_token({access_token, refresh_token, expires_at}) do with true <- needs_to_refresh_token?(expires_at), {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do {:ok, new_access_token} diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex new file mode 100644 index 000000000..f9062d00e --- /dev/null +++ b/lib/plausible/google/ga4/api.ex @@ -0,0 +1,142 @@ +defmodule Plausible.Google.GA4.API do + @moduledoc """ + API for Google Analytics 4. + """ + + alias Plausible.Google + alias Plausible.Google.GA4 + + require Logger + + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + + @per_page 50_000 + @backoff_factor :timer.seconds(10) + @max_attempts 5 + + def list_properties(access_token) do + case GA4.HTTP.list_accounts_for_user(access_token) do + {:ok, %{"accountSummaries" => accounts}} -> + accounts = + accounts + |> Enum.filter(& &1["propertySummaries"]) + |> Enum.map(fn account -> + {"#{account["displayName"]} (#{account["account"]})", + Enum.map(account["propertySummaries"], fn property -> + {"#{property["displayName"]} (#{property["property"]})", property["property"]} + end)} + end) + + {:ok, accounts} + + error -> + error + end + end + + def get_property(access_token, lookup_property) do + case list_properties(access_token) do + {:ok, properties} -> + property = + properties + |> Enum.map(&elem(&1, 1)) + |> List.flatten() + |> Enum.find(fn {_name, property} -> property == lookup_property end) + + {:ok, property} + + {:error, cause} -> + {:error, cause} + end + end + + def get_analytics_start_date(access_token, property) do + GA4.HTTP.get_analytics_start_date(access_token, property) + end + + def import_analytics(date_range, property, auth, persist_fn) do + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" + ) + + with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do + do_import_analytics(date_range, property, access_token, persist_fn) + end + end + + defp do_import_analytics(date_range, property, access_token, persist_fn) do + Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}" + ) + + report_request = prepare_request(report_request, date_range, property, access_token) + + case fetch_and_persist(report_request, persist_fn: persist_fn) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} + def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + attempt = Keyword.get(opts, :attempt, 1) + sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) + + case GA4.HTTP.get_report(report_request) do + {:ok, {rows, row_count}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}" + ) + + :ok = persist_fn.(report_request.dataset, rows) + + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}" + ) + + if report_request.offset + @per_page < row_count do + fetch_and_persist( + %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page}, + opts + ) + else + :ok + end + + {:error, cause} -> + if attempt >= @max_attempts do + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating." + ) + + {:error, cause} + else + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry." + ) + + Process.sleep(attempt * sleep_time) + fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp prepare_request(report_request, date_range, property, access_token) do + %GA4.ReportRequest{ + report_request + | date_range: date_range, + property: property, + access_token: access_token, + offset: 0, + limit: @per_page + } + end +end diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex new file mode 100644 index 000000000..da9ace19b --- /dev/null +++ b/lib/plausible/google/ga4/http.ex @@ -0,0 +1,199 @@ +defmodule Plausible.Google.GA4.HTTP do + @moduledoc """ + HTTP client implementation for Google Analytics 4 API. + """ + + alias Plausible.HTTPClient + + require Logger + + @spec get_report(Plausible.Google.GA4.ReportRequest.t()) :: + {:ok, {[map()], non_neg_integer()}} | {:error, any()} + def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do + params = %{ + requests: [ + %{ + property: report_request.property, + dateRanges: [ + %{ + startDate: report_request.date_range.first, + endDate: report_request.date_range.last + } + ], + dimensions: Enum.map(report_request.dimensions, &%{name: &1}), + metrics: Enum.map(report_request.metrics, &build_metric/1), + orderBys: [ + %{ + dimension: %{ + dimensionName: "date", + orderType: "ALPHANUMERIC" + }, + desc: true + } + ], + limit: report_request.limit, + offset: report_request.offset + } + ] + } + + url = + "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports" + + response = + HTTPClient.impl().post( + url, + [{"Authorization", "Bearer #{report_request.access_token}"}], + params, + receive_timeout: 60_000 + ) + + with {:ok, %{body: body}} <- response, + # File.write!("fixture/ga4_report_#{report_request.dataset}.json", Jason.encode!(body)), + {:ok, report} <- parse_report_from_response(body), + row_count <- Map.get(report, "rowCount"), + {:ok, report} <- convert_to_maps(report) do + {:ok, {report, row_count}} + else + {:error, %{reason: %{status: status, body: body}}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}" + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + {:error, :request_failed} + + {:error, reason} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}" + ) + + {:error, :request_failed} + end + end + + defp build_metric(expression) do + case String.split(expression, " = ") do + [name, expression] -> + %{ + name: name, + expression: expression + } + + [name] -> + %{name: name} + end + end + + defp parse_report_from_response(%{"reports" => [report | _]}) do + {:ok, report} + end + + defp parse_report_from_response(body) do + Sentry.Context.set_extra_context(%{google_analytics4_response: body}) + + Logger.error( + "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}" + ) + + {:error, {:invalid_response, body}} + end + + defp convert_to_maps(%{ + "rows" => rows, + "dimensionHeaders" => dimension_headers, + "metricHeaders" => metric_headers + }) + when is_list(rows) do + dimension_headers = Enum.map(dimension_headers, & &1["name"]) + metric_headers = Enum.map(metric_headers, & &1["name"]) + + report = + Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} -> + dimension_values = Enum.map(dimensions, & &1["value"]) + metric_values = Enum.map(metrics, & &1["value"]) + metrics = Enum.zip(metric_headers, metric_values) + dimensions = Enum.zip(dimension_headers, dimension_values) + %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} + end) + + {:ok, report} + end + + defp convert_to_maps(response) do + Logger.error( + "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}" + ) + + Sentry.Context.set_extra_context(%{google_analytics4_response: response}) + {:error, {:invalid_response, response}} + end + + def list_accounts_for_user(access_token) do + url = "#{admin_api_url()}/v1beta/accountSummaries" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error}) + {:error, :unknown} + end + end + + @earliest_valid_date "2015-08-14" + def get_analytics_start_date(access_token, property) do + params = %{ + requests: [ + %{ + property: "#{property}", + dateRanges: [ + %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} + ], + dimensions: [%{name: "date"}], + metrics: [%{name: "screenPageViews"}], + orderBys: [ + %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false} + ], + limit: 1 + } + ] + } + + url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports" + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + report = List.first(body["reports"]) + + date = + case report["rows"] do + [%{"dimensionValues" => [%{"value" => date_str}]}] -> + Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() + + _ -> + nil + end + + {:ok, date} + + {:error, %{reason: %Finch.Response{body: body}}} -> + Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)}) + {:error, body} + + {:error, %{reason: reason} = e} -> + Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)}) + {:error, reason} + end + end + + defp reporting_api_url, do: "https://analyticsdata.googleapis.com" + defp admin_api_url, do: "https://analyticsadmin.googleapis.com" +end diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex new file mode 100644 index 000000000..b859f1789 --- /dev/null +++ b/lib/plausible/google/ga4/report_request.ex @@ -0,0 +1,122 @@ +defmodule Plausible.Google.GA4.ReportRequest do + @moduledoc """ + Report request struct for Google Analytics 4 API + """ + + defstruct [ + :dataset, + :dimensions, + :metrics, + :date_range, + :property, + :access_token, + :offset, + :limit + ] + + @type t() :: %__MODULE__{ + dataset: String.t(), + dimensions: [String.t()], + metrics: [String.t()], + date_range: Date.Range.t(), + property: term(), + access_token: String.t(), + offset: non_neg_integer(), + limit: non_neg_integer() + } + + def full_report do + [ + %__MODULE__{ + dataset: "imported_visitors", + dimensions: ["date"], + metrics: [ + "totalUsers", + "screenPageViews", + "bounces = sessions - engagedSessions", + "sessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_sources", + dimensions: [ + "date", + "sessionSource", + "sessionMedium", + "sessionCampaignName", + "sessionManualAdContent", + "sessionGoogleAdsKeyword" + ], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_pages", + dimensions: ["date", "hostName", "pagePath"], + # NOTE: no exits as GA4 DATA API does not provide that metric + metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"] + }, + %__MODULE__{ + dataset: "imported_entry_pages", + dimensions: ["date", "landingPage"], + metrics: [ + "totalUsers", + "sessions", + "userEngagementDuration", + "bounces = sessions - engagedSessions" + ] + }, + # NOTE: Skipping for now as there's no dimension directly mapping to exit page path + # %__MODULE__{ + # dataset: "imported_exit_pages", + # dimensions: ["date", "ga:exitPagePath"], + # metrics: ["totalUsers", "sessions"] + # }, + %__MODULE__{ + dataset: "imported_locations", + dimensions: ["date", "countryId", "region", "city"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_devices", + dimensions: ["date", "deviceCategory"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_browsers", + dimensions: ["date", "browser"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_operating_systems", + dimensions: ["date", "operatingSystem"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + } + ] + end +end diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex index 93dad5910..be55fbcf5 100644 --- a/lib/plausible/google/http.ex +++ b/lib/plausible/google/http.ex @@ -2,97 +2,6 @@ defmodule Plausible.Google.HTTP do require Logger alias Plausible.HTTPClient - @spec get_report(Plausible.Google.ReportRequest.t()) :: - {:ok, {[map()], String.t() | nil}} | {:error, any()} - def get_report(%Plausible.Google.ReportRequest{} = report_request) do - params = %{ - reportRequests: [ - %{ - viewId: report_request.view_id, - dateRanges: [ - %{ - startDate: report_request.date_range.first, - endDate: report_request.date_range.last - } - ], - dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}), - metrics: Enum.map(report_request.metrics, &%{expression: &1}), - hideTotals: true, - hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], - pageSize: report_request.page_size, - pageToken: report_request.page_token - } - ] - } - - response = - HTTPClient.impl().post( - "#{reporting_api_url()}/v4/reports:batchGet", - [{"Authorization", "Bearer #{report_request.access_token}"}], - params, - receive_timeout: 60_000 - ) - - with {:ok, %{body: body}} <- response, - {:ok, report} <- parse_report_from_response(body), - token <- Map.get(report, "nextPageToken"), - {:ok, report} <- convert_to_maps(report) do - {:ok, {report, token}} - else - {:error, %{reason: %{status: status, body: body}}} -> - Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) - {:error, :request_failed} - - {:error, _reason} -> - {:error, :request_failed} - end - end - - defp parse_report_from_response(body) do - with %{"reports" => [report | _]} <- body do - {:ok, report} - else - _ -> - Sentry.Context.set_extra_context(%{google_analytics_response: body}) - - Logger.error( - "Google Analytics: Failed to find report in response. Reason: #{inspect(body)}" - ) - - {:error, {:invalid_response, body}} - end - end - - defp convert_to_maps(%{ - "data" => %{} = data, - "columnHeader" => %{ - "dimensions" => dimension_headers, - "metricHeader" => %{"metricHeaderEntries" => metric_headers} - } - }) do - metric_headers = Enum.map(metric_headers, & &1["name"]) - rows = Map.get(data, "rows", []) - - report = - Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} -> - metrics = Enum.zip(metric_headers, metrics) - dimensions = Enum.zip(dimension_headers, dimensions) - %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} - end) - - {:ok, report} - end - - defp convert_to_maps(response) do - Logger.error( - "Google Analytics: Failed to read report in response. Reason: #{inspect(response)}" - ) - - Sentry.Context.set_extra_context(%{google_analytics_response: response}) - {:error, {:invalid_response, response}} - end - def list_sites(access_token) do url = "#{api_url()}/webmasters/v3/sites" headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}] @@ -113,7 +22,7 @@ defmodule Plausible.Google.HTTP do end end - def fetch_access_token(code) do + def fetch_access_token!(code) do url = "#{api_url()}/oauth2/v4/token" headers = [{"Content-Type", "application/x-www-form-urlencoded"}] @@ -130,24 +39,6 @@ defmodule Plausible.Google.HTTP do response.body end - def list_views_for_user(access_token) do - url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles" - - headers = [{"Authorization", "Bearer #{access_token}"}] - - case HTTPClient.impl().get(url, headers) do - {:ok, %Finch.Response{body: body, status: 200}} -> - {:ok, body} - - {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> - {:error, :authentication_failed} - - {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) - {:error, :unknown} - end - end - def list_stats(access_token, property, date_range, limit, page \\ nil) do property = URI.encode_www_form(property) @@ -215,57 +106,9 @@ defmodule Plausible.Google.HTTP do end end - @earliest_valid_date "2005-01-01" - def get_analytics_start_date(view_id, access_token) do - params = %{ - reportRequests: [ - %{ - viewId: view_id, - dateRanges: [ - %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} - ], - dimensions: [%{name: "ga:date", histogramBuckets: []}], - metrics: [%{expression: "ga:pageviews"}], - hideTotals: true, - hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], - pageSize: 1 - } - ] - } - - url = "#{reporting_api_url()}/v4/reports:batchGet" - headers = [{"Authorization", "Bearer #{access_token}"}] - - case HTTPClient.post(url, headers, params) do - {:ok, %Finch.Response{body: body, status: 200}} -> - report = List.first(body["reports"]) - - date = - case report["data"]["rows"] do - [%{"dimensions" => [date_str]}] -> - Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() - - _ -> - nil - end - - {:ok, date} - - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)}) - {:error, body} - - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)}) - {:error, reason} - end - end - defp config, do: Application.get_env(:plausible, :google) defp client_id, do: Keyword.fetch!(config(), :client_id) defp client_secret, do: Keyword.fetch!(config(), :client_secret) - defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url) defp api_url, do: Keyword.fetch!(config(), :api_url) defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback" end diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex new file mode 100644 index 000000000..47d2cebeb --- /dev/null +++ b/lib/plausible/google/ua/api.ex @@ -0,0 +1,146 @@ +defmodule Plausible.Google.UA.API do + @moduledoc """ + API for Universal Analytics + """ + + alias Plausible.Google + alias Plausible.Google.UA + + @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()} + + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + + @per_page 7_500 + @backoff_factor :timer.seconds(10) + @max_attempts 5 + + @spec list_views(access_token :: String.t()) :: + {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()} + @doc """ + Lists Google Analytics views grouped by hostname. + """ + def list_views(access_token) do + case UA.HTTP.list_views_for_user(access_token) do + {:ok, %{"items" => views}} -> + views = Enum.group_by(views, &view_hostname/1, &view_names/1) + {:ok, views} + + error -> + error + end + end + + @spec get_view(access_token :: String.t(), lookup_id :: String.t()) :: + {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()} + @doc """ + Returns a single Google Analytics view if the user has access to it. + """ + def get_view(access_token, lookup_id) do + case list_views(access_token) do + {:ok, views} -> + view = + views + |> Map.values() + |> List.flatten() + |> Enum.find(fn {_name, id} -> id == lookup_id end) + + {:ok, view} + + {:error, cause} -> + {:error, cause} + end + end + + def get_analytics_start_date(access_token, view_id) do + UA.HTTP.get_analytics_start_date(access_token, view_id) + end + + @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: + :ok | {:error, term()} + @doc """ + Imports stats from a Google Analytics UA view to a Plausible site. + + This function fetches Google Analytics reports in batches of #{@per_page} per + request. The batches are then passed to persist callback. + + Requests to Google Analytics can fail, and are retried at most + #{@max_attempts} times with an exponential backoff. Returns `:ok` when + importing has finished or `{:error, term()}` when a request to GA failed too + many times. + + Useful links: + + - [Feature documentation](https://plausible.io/docs/google-analytics-import) + - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest) + - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) + + """ + def import_analytics(date_range, view_id, auth, persist_fn) do + with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do + do_import_analytics(date_range, view_id, access_token, persist_fn) + end + end + + defp do_import_analytics(date_range, view_id, access_token, persist_fn) do + Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok -> + report_request = %UA.ReportRequest{ + report_request + | date_range: date_range, + view_id: view_id, + access_token: access_token, + page_token: nil, + page_size: @per_page + } + + case fetch_and_persist(report_request, persist_fn: persist_fn) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + @spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} + def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + attempt = Keyword.get(opts, :attempt, 1) + sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) + + case UA.HTTP.get_report(report_request) do + {:ok, {rows, next_page_token}} -> + :ok = persist_fn.(report_request.dataset, rows) + + if next_page_token do + fetch_and_persist( + %UA.ReportRequest{report_request | page_token: next_page_token}, + opts + ) + else + :ok + end + + {:error, cause} -> + if attempt >= @max_attempts do + {:error, cause} + else + Process.sleep(attempt * sleep_time) + fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp view_hostname(view) do + case view do + %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host) + _any -> "Others" + end + end + + defp view_names(%{"name" => name, "id" => id}) do + {"#{id} - #{name}", id} + end +end diff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.ex new file mode 100644 index 000000000..f00712f5e --- /dev/null +++ b/lib/plausible/google/ua/http.ex @@ -0,0 +1,167 @@ +defmodule Plausible.Google.UA.HTTP do + @moduledoc """ + HTTP client implementation for Universal Analytics API. + """ + + require Logger + alias Plausible.HTTPClient + + @spec get_report(Plausible.Google.UA.ReportRequest.t()) :: + {:ok, {[map()], String.t() | nil}} | {:error, any()} + def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do + params = %{ + reportRequests: [ + %{ + viewId: report_request.view_id, + dateRanges: [ + %{ + startDate: report_request.date_range.first, + endDate: report_request.date_range.last + } + ], + dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(report_request.metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true, + orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], + pageSize: report_request.page_size, + pageToken: report_request.page_token + } + ] + } + + response = + HTTPClient.impl().post( + "#{reporting_api_url()}/v4/reports:batchGet", + [{"Authorization", "Bearer #{report_request.access_token}"}], + params, + receive_timeout: 60_000 + ) + + with {:ok, %{body: body}} <- response, + {:ok, report} <- parse_report_from_response(body), + token <- Map.get(report, "nextPageToken"), + {:ok, report} <- convert_to_maps(report) do + {:ok, {report, token}} + else + {:error, %{reason: %{status: status, body: body}}} -> + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + {:error, :request_failed} + + {:error, _reason} -> + {:error, :request_failed} + end + end + + defp parse_report_from_response(%{"reports" => [report | _]}) do + {:ok, report} + end + + defp parse_report_from_response(body) do + Sentry.Context.set_extra_context(%{universal_analytics_response: body}) + + Logger.error( + "Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}" + ) + + {:error, {:invalid_response, body}} + end + + defp convert_to_maps(%{ + "data" => %{} = data, + "columnHeader" => %{ + "dimensions" => dimension_headers, + "metricHeader" => %{"metricHeaderEntries" => metric_headers} + } + }) do + metric_headers = Enum.map(metric_headers, & &1["name"]) + rows = Map.get(data, "rows", []) + + report = + Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} -> + metrics = Enum.zip(metric_headers, metrics) + dimensions = Enum.zip(dimension_headers, dimensions) + %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} + end) + + {:ok, report} + end + + defp convert_to_maps(response) do + Logger.error( + "Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}" + ) + + Sentry.Context.set_extra_context(%{universal_analytics_response: response}) + {:error, {:invalid_response, response}} + end + + def list_views_for_user(access_token) do + url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) + {:error, :unknown} + end + end + + @earliest_valid_date "2005-01-01" + def get_analytics_start_date(access_token, view_id) do + params = %{ + reportRequests: [ + %{ + viewId: view_id, + dateRanges: [ + %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} + ], + dimensions: [%{name: "ga:date", histogramBuckets: []}], + metrics: [%{expression: "ga:pageviews"}], + hideTotals: true, + hideValueRanges: true, + orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], + pageSize: 1 + } + ] + } + + url = "#{reporting_api_url()}/v4/reports:batchGet" + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + report = List.first(body["reports"]) + + date = + case report["data"]["rows"] do + [%{"dimensions" => [date_str]}] -> + Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() + + _ -> + nil + end + + {:ok, date} + + {:error, %{reason: %Finch.Response{body: body}}} -> + Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)}) + {:error, body} + + {:error, %{reason: reason} = e} -> + Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)}) + {:error, reason} + end + end + + defp config, do: Application.get_env(:plausible, :google) + defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url) + defp api_url, do: Keyword.fetch!(config(), :api_url) +end diff --git a/lib/plausible/google/report_request.ex b/lib/plausible/google/ua/report_request.ex similarity index 95% rename from lib/plausible/google/report_request.ex rename to lib/plausible/google/ua/report_request.ex index fcc8fe2c1..e55419dd7 100644 --- a/lib/plausible/google/report_request.ex +++ b/lib/plausible/google/ua/report_request.ex @@ -1,4 +1,8 @@ -defmodule Plausible.Google.ReportRequest do +defmodule Plausible.Google.UA.ReportRequest do + @moduledoc """ + Report request struct for Universal Analytics API + """ + defstruct [ :dataset, :dimensions, diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex index 554cfaf70..fccbb6278 100644 --- a/lib/plausible/imported/buffer.ex +++ b/lib/plausible/imported/buffer.ex @@ -8,12 +8,13 @@ defmodule Plausible.Imported.Buffer do use GenServer require Logger - def start_link do - GenServer.start_link(__MODULE__, nil) + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts) end - def init(_opts) do - {:ok, %{buffers: %{}}} + def init(opts) do + flush_interval = Keyword.get(opts, :flush_interval_ms, 1000) + {:ok, %{flush_interval: flush_interval, buffers: %{}}} end @spec insert_many(pid(), term(), [map()]) :: :ok @@ -68,14 +69,14 @@ defmodule Plausible.Imported.Buffer do def handle_call(:flush_all_buffers, _from, state) do Enum.each(state.buffers, fn {table_name, records} -> - flush_buffer(records, table_name) + flush_buffer(records, table_name, state.flush_interval) end) {:reply, :ok, put_in(state.buffers, %{})} end def handle_continue({:flush, table_name}, state) do - flush_buffer(state.buffers[table_name], table_name) + flush_buffer(state.buffers[table_name], table_name, state.flush_interval) {:noreply, put_in(state.buffers[table_name], [])} end @@ -85,10 +86,10 @@ defmodule Plausible.Imported.Buffer do |> Keyword.fetch!(:max_buffer_size) end - defp flush_buffer(records, table_name) do + defp flush_buffer(records, table_name, flush_interval) do # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this # sleep call slows down the flushing - Process.sleep(1000) + Process.sleep(flush_interval) Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer") insert_all(table_name, records) diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex new file mode 100644 index 000000000..583242133 --- /dev/null +++ b/lib/plausible/imported/google_analytics4.ex @@ -0,0 +1,253 @@ +defmodule Plausible.Imported.GoogleAnalytics4 do + @moduledoc """ + Import implementation for Google Analytics 4. + """ + + use Plausible.Imported.Importer + + @missing_values ["(none)", "(not set)", "(not provided)", "(other)"] + + @impl true + def name(), do: :google_analytics_4 + + @impl true + def label(), do: "Google Analytics 4" + + @impl true + def email_template(), do: "google_analytics_import.html" + + @impl true + def parse_args( + %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args + ) do + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + date_range = Date.range(start_date, end_date) + + auth = { + Map.fetch!(args, "access_token"), + Map.fetch!(args, "refresh_token"), + Map.fetch!(args, "token_expires_at") + } + + [ + property: property, + date_range: date_range, + auth: auth + ] + end + + @doc """ + Imports stats from a Google Analytics 4 property to a Plausible site. + + This function fetches Google Analytics 4 reports which are then passed in batches + to Clickhouse by the `Plausible.Imported.Buffer` process. + """ + @impl true + def import_data(site_import, opts) do + date_range = Keyword.fetch!(opts, :date_range) + property = Keyword.fetch!(opts, :property) + auth = Keyword.fetch!(opts, :auth) + flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000) + + {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms) + + persist_fn = fn table, rows -> + records = from_report(rows, site_import.site_id, site_import.id, table) + Plausible.Imported.Buffer.insert_many(buffer, table, records) + end + + try do + Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn) + after + Plausible.Imported.Buffer.flush(buffer) + Plausible.Imported.Buffer.stop(buffer) + end + end + + def from_report(nil, _site_id, _import_id, _metric), do: nil + + def from_report(data, site_id, import_id, table) do + Enum.reduce(data, [], fn row, acc -> + if Map.get(row.dimensions, "date") in @missing_values do + acc + else + [new_from_report(site_id, import_id, table, row) | acc] + end + end) + end + + defp parse_number(nr) do + {float, ""} = Float.parse(nr) + round(float) + end + + defp new_from_report(site_id, import_id, "imported_visitors", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_sources", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(), + utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(), + utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(), + utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(), + utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""), + page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + # NOTE: no exits metric in GA4 API currently + exits: 0, + time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_entry_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + entry_page: row.dimensions |> Map.fetch!("landingPage"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() + } + end + + # NOTE: no exit pages metrics in GA4 API available for now + # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do + # %{ + # site_id: site_id, + # import_id: import_id, + # date: get_date(row), + # exit_page: Map.fetch!(row.dimensions, "exitPage"), + # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + # exits: row.metrics |> Map.fetch!("sessions") |> parse_number() + # } + # end + + defp new_from_report(site_id, import_id, "imported_locations", row) do + country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("") + city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("") + city_data = Location.get_city(city_name, country_code) + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + country: country_code, + region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""), + city: city_data && city_data.id, + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_devices", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @browser_google_to_plausible %{ + "User-Agent:Opera" => "Opera", + "Mozilla Compatible Agent" => "Mobile App", + "Android Webview" => "Mobile App", + "Android Browser" => "Mobile App", + "Safari (in-app)" => "Mobile App", + "User-Agent: Mozilla" => "Firefox", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_browsers", row) do + browser = Map.fetch!(row.dimensions, "browser") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + browser: Map.get(@browser_google_to_plausible, browser, browser), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @os_google_to_plausible %{ + "Macintosh" => "Mac", + "Linux" => "GNU/Linux", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_operating_systems", row) do + os = Map.fetch!(row.dimensions, "operatingSystem") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + operating_system: Map.get(@os_google_to_plausible, os, os), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp get_date(%{dimensions: %{"date" => date}}) do + date + |> Timex.parse!("%Y%m%d", :strftime) + |> NaiveDateTime.to_date() + end + + defp default_if_missing(value, default \\ nil) + defp default_if_missing(value, default) when value in @missing_values, do: default + defp default_if_missing(value, _default), do: value + + defp parse_referrer(nil), do: nil + defp parse_referrer("(direct)"), do: nil + defp parse_referrer("google"), do: "Google" + defp parse_referrer("bing"), do: "Bing" + defp parse_referrer("duckduckgo"), do: "DuckDuckGo" + + defp parse_referrer(ref) do + RefInspector.parse("https://" <> ref) + |> PlausibleWeb.RefInspector.parse() + end +end diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.ex index bcf1eb8d4..96a149b19 100644 --- a/lib/plausible/imported/import_sources.ex +++ b/lib/plausible/imported/import_sources.ex @@ -4,6 +4,7 @@ defmodule Plausible.Imported.ImportSources do """ @sources [ + Plausible.Imported.GoogleAnalytics4, Plausible.Imported.UniversalAnalytics, Plausible.Imported.NoopImporter, Plausible.Imported.CSVImporter diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex index 1c7a1e093..1d0dcb2e8 100644 --- a/lib/plausible/imported/universal_analytics.ex +++ b/lib/plausible/imported/universal_analytics.ex @@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics do end try do - Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn) + Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn) after Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.stop(buffer) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 11a68efc5..a24241371 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -697,13 +697,16 @@ defmodule PlausibleWeb.AuthController do end def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do - [site_id, _redirected_to, legacy] = + [site_id, _redirected_to, legacy, _ga4] = case Jason.decode!(state) do [site_id, redirect_to] -> - [site_id, redirect_to, true] + [site_id, redirect_to, true, false] [site_id, redirect_to, legacy] -> - [site_id, redirect_to, legacy] + [site_id, redirect_to, legacy, false] + + [site_id, redirect_to, legacy, ga4] -> + [site_id, redirect_to, legacy, ga4] end site = Repo.get(Plausible.Site, site_id) @@ -745,15 +748,18 @@ defmodule PlausibleWeb.AuthController do end def google_auth_callback(conn, %{"code" => code, "state" => state}) do - res = Plausible.Google.HTTP.fetch_access_token(code) + res = Plausible.Google.API.fetch_access_token!(code) - [site_id, redirect_to, legacy] = + [site_id, redirect_to, legacy, ga4] = case Jason.decode!(state) do [site_id, redirect_to] -> - [site_id, redirect_to, true] + [site_id, redirect_to, true, false] [site_id, redirect_to, legacy] -> - [site_id, redirect_to, legacy] + [site_id, redirect_to, legacy, false] + + [site_id, redirect_to, legacy, ga4] -> + [site_id, redirect_to, legacy, ga4] end site = Repo.get(Plausible.Site, site_id) @@ -761,15 +767,26 @@ defmodule PlausibleWeb.AuthController do case redirect_to do "import" -> - redirect(conn, - external: - Routes.site_path(conn, :import_from_google_view_id_form, site.domain, - access_token: res["access_token"], - refresh_token: res["refresh_token"], - expires_at: NaiveDateTime.to_iso8601(expires_at), - legacy: legacy - ) - ) + if ga4 do + redirect(conn, + external: + Routes.google_analytics4_path(conn, :property_form, site.domain, + access_token: res["access_token"], + refresh_token: res["refresh_token"], + expires_at: NaiveDateTime.to_iso8601(expires_at) + ) + ) + else + redirect(conn, + external: + Routes.universal_analytics_path(conn, :view_id_form, site.domain, + access_token: res["access_token"], + refresh_token: res["refresh_token"], + expires_at: NaiveDateTime.to_iso8601(expires_at), + legacy: legacy + ) + ) + end _ -> id_token = res["id_token"] diff --git a/lib/plausible_web/controllers/google_analytics4_controller.ex b/lib/plausible_web/controllers/google_analytics4_controller.ex new file mode 100644 index 000000000..3fd55d3ab --- /dev/null +++ b/lib/plausible_web/controllers/google_analytics4_controller.ex @@ -0,0 +1,143 @@ +defmodule PlausibleWeb.GoogleAnalytics4Controller do + use PlausibleWeb, :controller + + plug(PlausibleWeb.RequireAccountPlug) + + plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) + + def property_form(conn, %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + redirect_route = Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) + + case Plausible.Google.GA4.API.list_properties(access_token) do + {:ok, properties} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("property_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: conn.assigns.site, + properties: properties, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + ) + |> redirect(external: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end + end + + def property(conn, %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property) + + case start_date do + {:ok, nil} -> + {:ok, properties} = Plausible.Google.GA4.API.list_properties(access_token) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("property_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + properties: properties, + selected_property_error: "No data found. Nothing to import", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:ok, _date} -> + redirect(conn, + to: + Routes.google_analytics4_path(conn, :confirm, site.domain, + property: property, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at + ) + ) + end + end + + def confirm(conn, %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + + start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property) + + end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + + {:ok, {property_name, property}} = + Plausible.Google.GA4.API.get_property(access_token, property) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_property: property, + selected_property_name: property_name, + start_date: start_date, + end_date: end_date, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import(conn, %{ + "property" => property, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + current_user = conn.assigns.current_user + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + {:ok, _} = + Plausible.Imported.GoogleAnalytics4.new_import( + site, + current_user, + property: property, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at + ) + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 51c04c799..aa38531cc 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -232,7 +232,7 @@ defmodule PlausibleWeb.SiteController do search_console_domains = if site.google_auth do - Plausible.Google.Api.fetch_verified_properties(site.google_auth) + Plausible.Google.API.fetch_verified_properties(site.google_auth) end imported_pageviews = @@ -641,198 +641,6 @@ defmodule PlausibleWeb.SiteController do end end - def import_from_google_user_metric_notice(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_user_metric_form.html", - site: site, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - end - - def import_from_google_view_id_form(conn, %{ - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - redirect_route = - if legacy == "true" do - Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain) - else - Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) - end - - case Plausible.Google.Api.list_views(access_token) do - {:ok, view_ids} -> - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_view_id_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: conn.assigns.site, - view_ids: view_ids, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - - {:error, :authentication_failed} -> - conn - |> put_flash( - :error, - "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." - ) - |> redirect(external: redirect_route) - - {:error, _any} -> - conn - |> put_flash( - :error, - "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." - ) - |> redirect(external: redirect_route) - end - end - - # see https://stackoverflow.com/a/57416769 - @google_analytics_new_user_metric_date ~D[2016-08-24] - def import_from_google_view_id(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) - - case start_date do - {:ok, nil} -> - site = conn.assigns[:site] - {:ok, view_ids} = Plausible.Google.Api.list_views(access_token) - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_view_id_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - view_ids: view_ids, - selected_view_id_error: "No data found. Nothing to import", - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - - {:ok, date} -> - if Timex.before?(date, @google_analytics_new_user_metric_date) do - redirect(conn, - to: - Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) - ) - else - redirect(conn, - to: - Routes.site_path(conn, :import_from_google_confirm, site.domain, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) - ) - end - end - end - - def import_from_google_confirm(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - - start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) - - end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) - - {:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id) - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_confirm.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - selected_view_id: view_id, - selected_view_id_name: view_name, - start_date: start_date, - end_date: end_date, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - end - - def import_from_google(conn, %{ - "view_id" => view_id, - "start_date" => start_date, - "end_date" => end_date, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns.site - current_user = conn.assigns.current_user - - redirect_route = - if legacy == "true" do - Routes.site_path(conn, :settings_integrations, site.domain) - else - Routes.site_path(conn, :settings_imports_exports, site.domain) - end - - {:ok, _} = - Plausible.Imported.UniversalAnalytics.new_import( - site, - current_user, - view_id: view_id, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at, - legacy: legacy == "true" - ) - - conn - |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") - |> redirect(external: redirect_route) - end - def forget_import(conn, %{"import_id" => import_id}) do site = conn.assigns.site diff --git a/lib/plausible_web/controllers/universal_analytics_controller.ex b/lib/plausible_web/controllers/universal_analytics_controller.ex new file mode 100644 index 000000000..16a4bea64 --- /dev/null +++ b/lib/plausible_web/controllers/universal_analytics_controller.ex @@ -0,0 +1,198 @@ +defmodule PlausibleWeb.UniversalAnalyticsController do + use PlausibleWeb, :controller + + plug(PlausibleWeb.RequireAccountPlug) + + plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) + + def user_metric_notice(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + + conn + |> assign(:skip_plausible_tracking, true) + |> render("user_metric_form.html", + site: site, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def view_id_form(conn, %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain) + else + Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) + end + + case Plausible.Google.UA.API.list_views(access_token) do + {:ok, view_ids} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("view_id_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: conn.assigns.site, + view_ids: view_ids, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + ) + |> redirect(external: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end + end + + # see https://stackoverflow.com/a/57416769 + @google_analytics_new_user_metric_date ~D[2016-08-24] + def view_id(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id) + + case start_date do + {:ok, nil} -> + {:ok, view_ids} = Plausible.Google.UA.API.list_views(access_token) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("view_id_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + view_ids: view_ids, + selected_view_id_error: "No data found. Nothing to import", + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:ok, date} -> + if Timex.before?(date, @google_analytics_new_user_metric_date) do + redirect(conn, + to: + Routes.universal_analytics_path(conn, :user_metric_notice, site.domain, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy + ) + ) + else + redirect(conn, + to: + Routes.universal_analytics_path(conn, :confirm, site.domain, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy + ) + ) + end + end + end + + def confirm(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + + start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id) + + end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + + {:ok, {view_name, view_id}} = Plausible.Google.UA.API.get_view(access_token, view_id) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_view_id: view_id, + selected_view_id_name: view_name, + start_date: start_date, + end_date: end_date, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import(conn, %{ + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + current_user = conn.assigns.current_user + + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + {:ok, _} = + Plausible.Imported.UniversalAnalytics.new_import( + site, + current_user, + view_id: view_id, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at, + legacy: legacy == "true" + ) + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index d1c414ac7..162dee3ca 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -374,17 +374,27 @@ defmodule PlausibleWeb.Router do delete "/:website/stats", SiteController, :reset_stats get "/:website/import/google-analytics/view-id", - SiteController, - :import_from_google_view_id_form + UniversalAnalyticsController, + :view_id_form - post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id + post "/:website/import/google-analytics/view-id", UniversalAnalyticsController, :view_id get "/:website/import/google-analytics/user-metric", - SiteController, - :import_from_google_user_metric_notice + UniversalAnalyticsController, + :user_metric_notice + + get "/:website/import/google-analytics/confirm", UniversalAnalyticsController, :confirm + post "/:website/settings/google-import", UniversalAnalyticsController, :import + + get "/:website/import/google-analytics4/property", + GoogleAnalytics4Controller, + :property_form + + post "/:website/import/google-analytics4/property", GoogleAnalytics4Controller, :property + + get "/:website/import/google-analytics4/confirm", GoogleAnalytics4Controller, :confirm + post "/:website/settings/google4-import", GoogleAnalytics4Controller, :import - get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm - post "/:website/settings/google-import", SiteController, :import_from_google delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import diff --git a/lib/plausible_web/templates/google_analytics4/confirm.html.heex b/lib/plausible_web/templates/google_analytics4/confirm.html.heex new file mode 100644 index 000000000..b0f63048b --- /dev/null +++ b/lib/plausible_web/templates/google_analytics4/confirm.html.heex @@ -0,0 +1,46 @@ +<%= form_for @conn, Routes.google_analytics4_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +
+ The following error occurred when fetching your Google Analytics 4 data. +
+<%= error %>
+ <% end %> + + <%= submit("Confirm import", class: "button mt-6") %> +<% end %> diff --git a/lib/plausible_web/templates/google_analytics4/property_form.html.heex b/lib/plausible_web/templates/google_analytics4/property_form.html.heex new file mode 100644 index 000000000..c88461073 --- /dev/null +++ b/lib/plausible_web/templates/google_analytics4/property_form.html.heex @@ -0,0 +1,19 @@ +<%= form_for @conn, Routes.google_analytics4_path(@conn, :property, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +- - - Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly. -
- -- The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you - can try importing again later. -
-The following error occurred when fetching your Google Analytics data.
++ The following error occurred when fetching your Google Analytics data. +
<%= error %>
- <% end %> + <% end %> - <%= submit "Confirm import", class: "button mt-6" %> + <%= submit("Confirm import", class: "button mt-6") %> <% end %> diff --git a/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex new file mode 100644 index 000000000..2f353f984 --- /dev/null +++ b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex @@ -0,0 +1,46 @@ ++ + Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly. +
+ ++ The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you + can try importing again later. +
+