From 1adda42a75c9bae556fc90a42ee52cfa5b0efe89 Mon Sep 17 00:00:00 2001 From: Vinicius Brasil Date: Mon, 26 Sep 2022 06:29:56 -0300 Subject: [PATCH] Refresh Google Analytics token before import (#2254) * Capture refresh and expires from GA callback * Pass GA refresh token to import worker * Refresh GA token before import --- lib/plausible/google/api.ex | 97 ++++++++++++++----- .../controllers/auth_controller.ex | 7 +- .../controllers/site_controller.ex | 48 +++++++-- .../site/import_from_google_confirm.html.eex | 2 + ...port_from_google_user_metric_form.html.eex | 2 +- .../import_from_google_view_id_form.html.eex | 2 + lib/workers/import_google_analytics.ex | 18 ++-- test/plausible/google/vcr_test.exs | 6 +- .../controllers/site_controller_test.exs | 18 +++- 9 files changed, 151 insertions(+), 49 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 85531658d..7878bc53f 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -30,8 +30,8 @@ defmodule Plausible.Google.Api do end def fetch_verified_properties(auth) do - with {:ok, auth} <- refresh_if_needed(auth), - {:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do + with {:ok, access_token} <- maybe_refresh_token(auth), + {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do sites |> Map.get("siteEntry", []) |> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end) @@ -43,10 +43,15 @@ defmodule Plausible.Google.Api do def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do with site <- Plausible.Repo.preload(site, :google_auth), - {:ok, %{access_token: access_token, property: property}} <- - refresh_if_needed(site.google_auth), + {:ok, access_token} <- maybe_refresh_token(site.google_auth), {:ok, stats} <- - HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do + HTTP.list_stats( + access_token, + site.google_auth.property, + date_range, + limit, + filters["page"] + ) do stats |> Map.get("rows", []) |> Enum.filter(fn row -> row["clicks"] > 0 end) @@ -103,9 +108,15 @@ defmodule Plausible.Google.Api do end end + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + @per_page 10_000 @max_attempts 5 - @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) :: + @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) :: :ok | {:error, term()} @doc """ Imports stats from a Google Analytics UA view to a Plausible site. @@ -125,7 +136,21 @@ defmodule Plausible.Google.Api do - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) """ - def import_analytics(site, date_range, view_id, access_token) do + def import_analytics(site, date_range, view_id, auth) do + with {:ok, access_token} <- maybe_refresh_token(auth), + :ok <- do_import_analytics(site, date_range, view_id, access_token) do + :ok + else + {:error, cause} -> + Sentry.capture_message("Failed to import from Google Analytics", + extra: %{site: site.domain, error: inspect(cause)} + ) + + {:error, cause} + end + end + + defp do_import_analytics(site, date_range, view_id, access_token) do {:ok, buffer} = Plausible.Google.Buffer.start_link() result = @@ -176,10 +201,6 @@ defmodule Plausible.Google.Api do {:error, cause} -> if attempt >= @max_attempts do - Sentry.capture_message("Failed to import from Google Analytics", - extra: %{site: site.domain, error: inspect(cause)} - ) - {:error, cause} else Process.sleep(sleep_time) @@ -188,28 +209,56 @@ defmodule Plausible.Google.Api do end end - defp refresh_if_needed(auth) do - if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do - do_refresh_token(auth) + defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do + with true <- needs_to_refresh_token?(auth.expires), + {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), + changeset <- + Plausible.Site.GoogleAuth.changeset(auth, %{ + access_token: new_access_token, + expires: expires_at + }), + {:ok, _google_auth} <- Plausible.Repo.update(changeset) do + {:ok, new_access_token} else - {:ok, auth} + false -> {:ok, auth.access_token} + {:error, cause} -> {:error, cause} end end - defp do_refresh_token(auth) do - case HTTP.refresh_auth_token(auth.refresh_token) do - {:ok, %{"access_token" => access_token, "expires_in" => expires_in}} -> - expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in) + defp maybe_refresh_token({access_token, nil, nil}) do + {:ok, access_token} + end + + defp maybe_refresh_token({access_token, refresh_token, expires_at}) do + if needs_to_refresh_token?(expires_at) do + do_refresh_token(refresh_token) + else + {:ok, access_token} + end + end - auth - |> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in}) - |> Plausible.Repo.update() + defp do_refresh_token(refresh_token) do + case HTTP.refresh_auth_token(refresh_token) do + {:ok, %{"access_token" => new_access_token, "expires_at" => expires_in}} -> + expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in) + {:ok, {new_access_token, expires_at}} - error -> - error + {:error, cause} -> + {:error, cause} end end + defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do + expires_at + |> NaiveDateTime.from_iso8601!() + |> needs_to_refresh_token?() + end + + defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do + thirty_seconds_ago = Timex.shift(Timex.now(), seconds: 30) + Timex.before?(expires_at, thirty_seconds_ago) + end + defp client_id() do Keyword.fetch!(Application.get_env(:plausible, :google), :client_id) end diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 29d9172a6..0dd6bfa42 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -546,13 +546,16 @@ defmodule PlausibleWeb.AuthController do res = Plausible.Google.HTTP.fetch_access_token(code) [site_id, redirect_to] = Jason.decode!(state) site = Repo.get(Plausible.Site, site_id) + expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"]) case redirect_to do "import" -> redirect(conn, to: Routes.site_path(conn, :import_from_google_view_id_form, site.domain, - access_token: res["access_token"] + access_token: res["access_token"], + refresh_token: res["refresh_token"], + expires_at: NaiveDateTime.to_iso8601(expires_at) ) ) @@ -565,7 +568,7 @@ defmodule PlausibleWeb.AuthController do email: id["email"], refresh_token: res["refresh_token"], access_token: res["access_token"], - expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]), + expires_at: expires_at, user_id: conn.assigns[:current_user].id, site_id: site_id }) diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 1dc855af5..d22af55b9 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -636,7 +636,9 @@ defmodule PlausibleWeb.SiteController do def import_from_google_user_metric_notice(conn, %{ "view_id" => view_id, - "access_token" => access_token + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at }) do site = conn.assigns[:site] @@ -646,11 +648,17 @@ defmodule PlausibleWeb.SiteController do site: site, view_id: view_id, access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, layout: {PlausibleWeb.LayoutView, "focus.html"} ) end - def import_from_google_view_id_form(conn, %{"access_token" => access_token}) do + def import_from_google_view_id_form(conn, %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do site = conn.assigns[:site] view_ids = Plausible.Google.Api.list_views(access_token) @@ -658,6 +666,8 @@ defmodule PlausibleWeb.SiteController do |> assign(:skip_plausible_tracking, true) |> render("import_from_google_view_id_form.html", access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, site: site, view_ids: view_ids, layout: {PlausibleWeb.LayoutView, "focus.html"} @@ -666,7 +676,12 @@ defmodule PlausibleWeb.SiteController do # see https://stackoverflow.com/a/57416769 @google_analytics_new_user_metric_date ~D[2016-08-24] - def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do + def import_from_google_view_id(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do site = conn.assigns[:site] start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) @@ -679,6 +694,8 @@ defmodule PlausibleWeb.SiteController do |> assign(:skip_plausible_tracking, true) |> render("import_from_google_view_id_form.html", access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, site: site, view_ids: view_ids, selected_view_id_error: "No data found. Nothing to import", @@ -691,7 +708,9 @@ defmodule PlausibleWeb.SiteController do to: Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain, view_id: view_id, - access_token: access_token + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at ) ) else @@ -699,14 +718,21 @@ defmodule PlausibleWeb.SiteController do to: Routes.site_path(conn, :import_from_google_confirm, site.domain, view_id: view_id, - access_token: access_token + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at ) ) end end end - def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do + def import_from_google_confirm(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do site = conn.assigns[:site] start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) @@ -720,6 +746,8 @@ defmodule PlausibleWeb.SiteController do |> assign(:skip_plausible_tracking, true) |> render("import_from_google_confirm.html", access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, site: site, selected_view_id: view_id, selected_view_id_name: view_name, @@ -733,7 +761,9 @@ defmodule PlausibleWeb.SiteController do "view_id" => view_id, "start_date" => start_date, "end_date" => end_date, - "access_token" => access_token + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at }) do site = conn.assigns[:site] @@ -743,7 +773,9 @@ defmodule PlausibleWeb.SiteController do "view_id" => view_id, "start_date" => start_date, "end_date" => end_date, - "access_token" => access_token + "access_token" => access_token, + "refresh_token" => refresh_token, + "token_expires_at" => expires_at }) Ecto.Multi.new() diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex index 072d998d1..4e1018faa 100644 --- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex +++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex @@ -2,6 +2,8 @@

Import from Google Analytics

<%= hidden_input(f, :access_token, value: @access_token) %> + <%= hidden_input(f, :refresh_token, value: @refresh_token) %> + <%= hidden_input(f, :expires_at, value: @expires_at) %> <%= case @start_date do %> <% {:ok, start_date} -> %> diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex index c303701f4..d8f69f2e5 100644 --- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex +++ b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex @@ -22,5 +22,5 @@

- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token), class: "button mt-6") %> + <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %> diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex index e6e9df24a..bd8bbee54 100644 --- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex +++ b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex @@ -2,6 +2,8 @@

Import from Google Analytics

<%= hidden_input(f, :access_token, value: @access_token) %> + <%= hidden_input(f, :refresh_token, value: @refresh_token) %> + <%= hidden_input(f, :expires_at, value: @expires_at) %> <%= case @view_ids do %> <% {:ok, view_ids} -> %> diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index df1dbeae1..0fc904374 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -9,13 +9,13 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do @impl Oban.Worker def perform( %Oban.Job{ - args: %{ - "site_id" => site_id, - "view_id" => view_id, - "start_date" => start_date, - "end_date" => end_date, - "access_token" => access_token - } + args: + %{ + "site_id" => site_id, + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date + } = args }, google_api \\ Plausible.Google.Api ) do @@ -24,7 +24,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do end_date = Date.from_iso8601!(end_date) date_range = Date.range(start_date, end_date) - case google_api.import_analytics(site, date_range, view_id, access_token) do + auth = {args["access_token"], args["refresh_token"], args["token_expires_at"]} + + case google_api.import_analytics(site, date_range, view_id, auth) do :ok -> Plausible.Site.import_success(site) |> Repo.update!() diff --git a/test/plausible/google/vcr_test.exs b/test/plausible/google/vcr_test.exs index cb41b6ddf..f93fe385d 100644 --- a/test/plausible/google/vcr_test.exs +++ b/test/plausible/google/vcr_test.exs @@ -21,11 +21,13 @@ defmodule Plausible.Google.Api.VCRTest do inserts_before_importing = get_insert_count() before_importing_timestamp = DateTime.utc_now() - access_token = "***" view_id = "54297898" date_range = Date.range(~D[2011-01-01], ~D[2022-07-19]) - assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token) + future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601() + auth = {"***", "refresh_token", future} + + assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, auth) total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second) total_inserts = get_insert_count() - inserts_before_importing diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index b4dc35ed8..f0d009064 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -745,7 +745,11 @@ defmodule PlausibleWeb.SiteControllerTest do response = conn - |> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token"}) + |> get("/#{site.domain}/import/google-analytics/view-id", %{ + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777" + }) |> html_response(200) assert response =~ "57238190 - one.test" @@ -761,7 +765,9 @@ defmodule PlausibleWeb.SiteControllerTest do "view_id" => "123", "start_date" => "2018-03-01", "end_date" => "2022-03-01", - "access_token" => "token" + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777" }) imported_data = Repo.reload(site).imported_data @@ -777,7 +783,9 @@ defmodule PlausibleWeb.SiteControllerTest do "view_id" => "123", "start_date" => "2018-03-01", "end_date" => "2022-03-01", - "access_token" => "token" + "access_token" => "token", + "refresh_token" => "foo", + "expires_at" => "2022-09-22T20:01:37.112777" }) assert_enqueued( @@ -787,7 +795,9 @@ defmodule PlausibleWeb.SiteControllerTest do "view_id" => "123", "start_date" => "2018-03-01", "end_date" => "2022-03-01", - "access_token" => "token" + "access_token" => "token", + "refresh_token" => "foo", + "token_expires_at" => "2022-09-22T20:01:37.112777" } ) end