From 1adda42a75c9bae556fc90a42ee52cfa5b0efe89 Mon Sep 17 00:00:00 2001
From: Vinicius Brasil
Date: Mon, 26 Sep 2022 06:29:56 -0300
Subject: [PATCH] Refresh Google Analytics token before import (#2254)
* Capture refresh and expires from GA callback
* Pass GA refresh token to import worker
* Refresh GA token before import
---
lib/plausible/google/api.ex | 97 ++++++++++++++-----
.../controllers/auth_controller.ex | 7 +-
.../controllers/site_controller.ex | 48 +++++++--
.../site/import_from_google_confirm.html.eex | 2 +
...port_from_google_user_metric_form.html.eex | 2 +-
.../import_from_google_view_id_form.html.eex | 2 +
lib/workers/import_google_analytics.ex | 18 ++--
test/plausible/google/vcr_test.exs | 6 +-
.../controllers/site_controller_test.exs | 18 +++-
9 files changed, 151 insertions(+), 49 deletions(-)
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 85531658d..7878bc53f 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -30,8 +30,8 @@ defmodule Plausible.Google.Api do
end
def fetch_verified_properties(auth) do
- with {:ok, auth} <- refresh_if_needed(auth),
- {:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do
+ with {:ok, access_token} <- maybe_refresh_token(auth),
+ {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do
sites
|> Map.get("siteEntry", [])
|> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
@@ -43,10 +43,15 @@ defmodule Plausible.Google.Api do
def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
with site <- Plausible.Repo.preload(site, :google_auth),
- {:ok, %{access_token: access_token, property: property}} <-
- refresh_if_needed(site.google_auth),
+ {:ok, access_token} <- maybe_refresh_token(site.google_auth),
{:ok, stats} <-
- HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do
+ HTTP.list_stats(
+ access_token,
+ site.google_auth.property,
+ date_range,
+ limit,
+ filters["page"]
+ ) do
stats
|> Map.get("rows", [])
|> Enum.filter(fn row -> row["clicks"] > 0 end)
@@ -103,9 +108,15 @@ defmodule Plausible.Google.Api do
end
end
+ @type import_auth :: {
+ access_token :: String.t(),
+ refresh_token :: String.t(),
+ expires_at :: String.t()
+ }
+
@per_page 10_000
@max_attempts 5
- @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) ::
+ @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
@@ -125,7 +136,21 @@ defmodule Plausible.Google.Api do
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
- def import_analytics(site, date_range, view_id, access_token) do
+ def import_analytics(site, date_range, view_id, auth) do
+ with {:ok, access_token} <- maybe_refresh_token(auth),
+ :ok <- do_import_analytics(site, date_range, view_id, access_token) do
+ :ok
+ else
+ {:error, cause} ->
+ Sentry.capture_message("Failed to import from Google Analytics",
+ extra: %{site: site.domain, error: inspect(cause)}
+ )
+
+ {:error, cause}
+ end
+ end
+
+ defp do_import_analytics(site, date_range, view_id, access_token) do
{:ok, buffer} = Plausible.Google.Buffer.start_link()
result =
@@ -176,10 +201,6 @@ defmodule Plausible.Google.Api do
{:error, cause} ->
if attempt >= @max_attempts do
- Sentry.capture_message("Failed to import from Google Analytics",
- extra: %{site: site.domain, error: inspect(cause)}
- )
-
{:error, cause}
else
Process.sleep(sleep_time)
@@ -188,28 +209,56 @@ defmodule Plausible.Google.Api do
end
end
- defp refresh_if_needed(auth) do
- if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do
- do_refresh_token(auth)
+ defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
+ with true <- needs_to_refresh_token?(auth.expires),
+ {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
+ changeset <-
+ Plausible.Site.GoogleAuth.changeset(auth, %{
+ access_token: new_access_token,
+ expires: expires_at
+ }),
+ {:ok, _google_auth} <- Plausible.Repo.update(changeset) do
+ {:ok, new_access_token}
else
- {:ok, auth}
+ false -> {:ok, auth.access_token}
+ {:error, cause} -> {:error, cause}
end
end
- defp do_refresh_token(auth) do
- case HTTP.refresh_auth_token(auth.refresh_token) do
- {:ok, %{"access_token" => access_token, "expires_in" => expires_in}} ->
- expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
+ defp maybe_refresh_token({access_token, nil, nil}) do
+ {:ok, access_token}
+ end
+
+ defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
+ if needs_to_refresh_token?(expires_at) do
+ do_refresh_token(refresh_token)
+ else
+ {:ok, access_token}
+ end
+ end
- auth
- |> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in})
- |> Plausible.Repo.update()
+ defp do_refresh_token(refresh_token) do
+ case HTTP.refresh_auth_token(refresh_token) do
+ {:ok, %{"access_token" => new_access_token, "expires_at" => expires_in}} ->
+ expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
+ {:ok, {new_access_token, expires_at}}
- error ->
- error
+ {:error, cause} ->
+ {:error, cause}
end
end
+ defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do
+ expires_at
+ |> NaiveDateTime.from_iso8601!()
+ |> needs_to_refresh_token?()
+ end
+
+ defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do
+ thirty_seconds_ago = Timex.shift(Timex.now(), seconds: 30)
+ Timex.before?(expires_at, thirty_seconds_ago)
+ end
+
defp client_id() do
Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
end
diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex
index 29d9172a6..0dd6bfa42 100644
--- a/lib/plausible_web/controllers/auth_controller.ex
+++ b/lib/plausible_web/controllers/auth_controller.ex
@@ -546,13 +546,16 @@ defmodule PlausibleWeb.AuthController do
res = Plausible.Google.HTTP.fetch_access_token(code)
[site_id, redirect_to] = Jason.decode!(state)
site = Repo.get(Plausible.Site, site_id)
+ expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])
case redirect_to do
"import" ->
redirect(conn,
to:
Routes.site_path(conn, :import_from_google_view_id_form, site.domain,
- access_token: res["access_token"]
+ access_token: res["access_token"],
+ refresh_token: res["refresh_token"],
+ expires_at: NaiveDateTime.to_iso8601(expires_at)
)
)
@@ -565,7 +568,7 @@ defmodule PlausibleWeb.AuthController do
email: id["email"],
refresh_token: res["refresh_token"],
access_token: res["access_token"],
- expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),
+ expires_at: expires_at,
user_id: conn.assigns[:current_user].id,
site_id: site_id
})
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 1dc855af5..d22af55b9 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -636,7 +636,9 @@ defmodule PlausibleWeb.SiteController do
def import_from_google_user_metric_notice(conn, %{
"view_id" => view_id,
- "access_token" => access_token
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
}) do
site = conn.assigns[:site]
@@ -646,11 +648,17 @@ defmodule PlausibleWeb.SiteController do
site: site,
view_id: view_id,
access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
- def import_from_google_view_id_form(conn, %{"access_token" => access_token}) do
+ def import_from_google_view_id_form(conn, %{
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
site = conn.assigns[:site]
view_ids = Plausible.Google.Api.list_views(access_token)
@@ -658,6 +666,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
site: site,
view_ids: view_ids,
layout: {PlausibleWeb.LayoutView, "focus.html"}
@@ -666,7 +676,12 @@ defmodule PlausibleWeb.SiteController do
# see https://stackoverflow.com/a/57416769
@google_analytics_new_user_metric_date ~D[2016-08-24]
- def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do
+ def import_from_google_view_id(conn, %{
+ "view_id" => view_id,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@@ -679,6 +694,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
site: site,
view_ids: view_ids,
selected_view_id_error: "No data found. Nothing to import",
@@ -691,7 +708,9 @@ defmodule PlausibleWeb.SiteController do
to:
Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,
view_id: view_id,
- access_token: access_token
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at
)
)
else
@@ -699,14 +718,21 @@ defmodule PlausibleWeb.SiteController do
to:
Routes.site_path(conn, :import_from_google_confirm, site.domain,
view_id: view_id,
- access_token: access_token
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at
)
)
end
end
end
- def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do
+ def import_from_google_confirm(conn, %{
+ "view_id" => view_id,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
@@ -720,6 +746,8 @@ defmodule PlausibleWeb.SiteController do
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_confirm.html",
access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
site: site,
selected_view_id: view_id,
selected_view_id_name: view_name,
@@ -733,7 +761,9 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
- "access_token" => access_token
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
}) do
site = conn.assigns[:site]
@@ -743,7 +773,9 @@ defmodule PlausibleWeb.SiteController do
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
- "access_token" => access_token
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "token_expires_at" => expires_at
})
Ecto.Multi.new()
diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
index 072d998d1..4e1018faa 100644
--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
@@ -2,6 +2,8 @@
Import from Google Analytics
<%= hidden_input(f, :access_token, value: @access_token) %>
+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+ <%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= case @start_date do %>
<% {:ok, start_date} -> %>
diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
index c303701f4..d8f69f2e5 100644
--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
@@ -22,5 +22,5 @@
- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token), class: "button mt-6") %>
+ <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %>
diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
index e6e9df24a..bd8bbee54 100644
--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
+++ b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
@@ -2,6 +2,8 @@
Import from Google Analytics
<%= hidden_input(f, :access_token, value: @access_token) %>
+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+ <%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= case @view_ids do %>
<% {:ok, view_ids} -> %>
diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex
index df1dbeae1..0fc904374 100644
--- a/lib/workers/import_google_analytics.ex
+++ b/lib/workers/import_google_analytics.ex
@@ -9,13 +9,13 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
@impl Oban.Worker
def perform(
%Oban.Job{
- args: %{
- "site_id" => site_id,
- "view_id" => view_id,
- "start_date" => start_date,
- "end_date" => end_date,
- "access_token" => access_token
- }
+ args:
+ %{
+ "site_id" => site_id,
+ "view_id" => view_id,
+ "start_date" => start_date,
+ "end_date" => end_date
+ } = args
},
google_api \\ Plausible.Google.Api
) do
@@ -24,7 +24,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do
end_date = Date.from_iso8601!(end_date)
date_range = Date.range(start_date, end_date)
- case google_api.import_analytics(site, date_range, view_id, access_token) do
+ auth = {args["access_token"], args["refresh_token"], args["token_expires_at"]}
+
+ case google_api.import_analytics(site, date_range, view_id, auth) do
:ok ->
Plausible.Site.import_success(site)
|> Repo.update!()
diff --git a/test/plausible/google/vcr_test.exs b/test/plausible/google/vcr_test.exs
index cb41b6ddf..f93fe385d 100644
--- a/test/plausible/google/vcr_test.exs
+++ b/test/plausible/google/vcr_test.exs
@@ -21,11 +21,13 @@ defmodule Plausible.Google.Api.VCRTest do
inserts_before_importing = get_insert_count()
before_importing_timestamp = DateTime.utc_now()
- access_token = "***"
view_id = "54297898"
date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
- assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token)
+ future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()
+ auth = {"***", "refresh_token", future}
+
+ assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, auth)
total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second)
total_inserts = get_insert_count() - inserts_before_importing
diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs
index b4dc35ed8..f0d009064 100644
--- a/test/plausible_web/controllers/site_controller_test.exs
+++ b/test/plausible_web/controllers/site_controller_test.exs
@@ -745,7 +745,11 @@ defmodule PlausibleWeb.SiteControllerTest do
response =
conn
- |> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token"})
+ |> get("/#{site.domain}/import/google-analytics/view-id", %{
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "expires_at" => "2022-09-22T20:01:37.112777"
+ })
|> html_response(200)
assert response =~ "57238190 - one.test"
@@ -761,7 +765,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
- "access_token" => "token"
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "expires_at" => "2022-09-22T20:01:37.112777"
})
imported_data = Repo.reload(site).imported_data
@@ -777,7 +783,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
- "access_token" => "token"
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "expires_at" => "2022-09-22T20:01:37.112777"
})
assert_enqueued(
@@ -787,7 +795,9 @@ defmodule PlausibleWeb.SiteControllerTest do
"view_id" => "123",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
- "access_token" => "token"
+ "access_token" => "token",
+ "refresh_token" => "foo",
+ "token_expires_at" => "2022-09-22T20:01:37.112777"
}
)
end