You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
analytics/lib/plausible/google/api.ex

181 lines
5.8 KiB
Elixir

defmodule Plausible.Google.Api do
alias Plausible.Google.{ReportRequest, HTTP}
use Timex
require Logger
@scope URI.encode_www_form(
"https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly"
)
@import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly")
@verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"]
def authorize_url(site_id, redirect_to) do
if Application.get_env(:plausible, :environment) == "test" do
""
else
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=" <>
Jason.encode!([site_id, redirect_to])
end
end
def import_authorize_url(site_id, redirect_to) do
if Application.get_env(:plausible, :environment) == "test" do
""
else
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
Jason.encode!([site_id, redirect_to])
end
end
def fetch_verified_properties(auth) do
with {:ok, auth} <- refresh_if_needed(auth),
{:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do
sites
|> Map.get("siteEntry", [])
|> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
|> Enum.map(fn site -> site["siteUrl"] end)
|> Enum.map(fn url -> String.trim_trailing(url, "/") end)
|> then(&{:ok, &1})
end
end
def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
with site <- Plausible.Repo.preload(site, :google_auth),
{:ok, %{access_token: access_token, property: property}} <-
refresh_if_needed(site.google_auth),
{:ok, stats} <-
HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do
stats
|> Map.get("rows", [])
|> Enum.filter(fn row -> row["clicks"] > 0 end)
|> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)
|> then(&{:ok, &1})
end
end
def get_analytics_view_ids(access_token) do
case HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
view_ids = for view <- views, do: build_view_ids(view), into: %{}
{:ok, view_ids}
error ->
error
end
end
defp build_view_ids(view) do
uri = URI.parse(Map.get(view, "websiteUrl", ""))
if !uri.host do
Sentry.capture_message("No URI for view ID", extra: view)
end
host = uri.host || Map.get(view, "id", "")
name = Map.get(view, "name")
{"#{host} - #{name}", Map.get(view, "id")}
end
@per_page 10_000
@one_day_in_ms 86_400_000
@doc """
API reference:
https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest
Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer
"""
def import_analytics(site, date_range, view_id, access_token) do
{:ok, buffer} = Plausible.Google.Buffer.start_link()
ReportRequest.full_report()
|> Task.async_stream(
fn %ReportRequest{} = report_request ->
report_request = %ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
fetch_and_persist(site, report_request, buffer: buffer)
end,
ordered: false,
max_concurrency: 3,
timeout: @one_day_in_ms
)
|> Stream.run()
Plausible.Google.Buffer.flush(buffer)
Plausible.Google.Buffer.stop(buffer)
:ok
end
@max_attempts 5
def fetch_and_persist(site, %ReportRequest{} = report_request, opts \\ []) do
buffer_pid = Keyword.get(opts, :buffer)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, 1000)
http_client = Keyword.get(opts, :http_client, Finch)
case HTTP.get_report(http_client, report_request) do
{:ok, {rows, next_page_token}} ->
records = Plausible.Imported.from_google_analytics(rows, site.id, report_request.dataset)
:ok = Plausible.Google.Buffer.insert_many(buffer_pid, report_request.dataset, records)
if next_page_token do
fetch_and_persist(
site,
%ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
error ->
context_key = "request:#{attempt}"
Sentry.Context.set_extra_context(%{context_key => error})
if attempt >= @max_attempts do
raise "Google API request failed too many times"
else
Process.sleep(sleep_time)
fetch_and_persist(site, report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp refresh_if_needed(auth) do
if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do
do_refresh_token(auth)
else
{:ok, auth}
end
end
defp do_refresh_token(auth) do
case HTTP.refresh_auth_token(auth.refresh_token) do
{:ok, %{"access_token" => access_token, "expires_in" => expires_in}} ->
expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
auth
|> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in})
|> Plausible.Repo.update()
error ->
error
end
end
defp client_id() do
Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
end
defp redirect_uri() do
PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
end
end