You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
438 lines
13 KiB
Elixir
438 lines
13 KiB
Elixir
defmodule Plausible.Google.Api do
|
|
alias Plausible.Imported
|
|
use Timex
|
|
require Logger
|
|
|
|
@scope URI.encode_www_form(
|
|
"https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly"
|
|
)
|
|
@import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly")
|
|
@verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"]
|
|
|
|
def authorize_url(site_id, redirect_to) do
|
|
if Application.get_env(:plausible, :environment) == "test" do
|
|
""
|
|
else
|
|
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=" <>
|
|
Jason.encode!([site_id, redirect_to])
|
|
end
|
|
end
|
|
|
|
def import_authorize_url(site_id, redirect_to) do
|
|
if Application.get_env(:plausible, :environment) == "test" do
|
|
""
|
|
else
|
|
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
|
|
Jason.encode!([site_id, redirect_to])
|
|
end
|
|
end
|
|
|
|
def fetch_access_token(code) do
|
|
res =
|
|
HTTPoison.post!(
|
|
"https://www.googleapis.com/oauth2/v4/token",
|
|
"client_id=#{client_id()}&client_secret=#{client_secret()}&code=#{code}&grant_type=authorization_code&redirect_uri=#{redirect_uri()}",
|
|
"Content-Type": "application/x-www-form-urlencoded"
|
|
)
|
|
|
|
Jason.decode!(res.body)
|
|
end
|
|
|
|
def fetch_verified_properties(auth) do
|
|
with {:ok, auth} <- refresh_if_needed(auth) do
|
|
res =
|
|
HTTPoison.get!("https://www.googleapis.com/webmasters/v3/sites",
|
|
"Content-Type": "application/json",
|
|
Authorization: "Bearer #{auth.access_token}"
|
|
)
|
|
|
|
domains =
|
|
Jason.decode!(res.body)
|
|
|> Map.get("siteEntry", [])
|
|
|> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
|
|
|> Enum.map(fn site -> site["siteUrl"] end)
|
|
|> Enum.map(fn url -> String.trim_trailing(url, "/") end)
|
|
|
|
{:ok, domains}
|
|
else
|
|
err -> err
|
|
end
|
|
end
|
|
|
|
defp property_base_url(property) do
|
|
case property do
|
|
"sc-domain:" <> domain -> "https://" <> domain
|
|
url -> url
|
|
end
|
|
end
|
|
|
|
def fetch_stats(site, query, limit) do
|
|
with {:ok, auth} <- refresh_if_needed(site.google_auth) do
|
|
do_fetch_stats(auth, query, limit)
|
|
else
|
|
err -> err
|
|
end
|
|
end
|
|
|
|
defp do_fetch_stats(auth, query, limit) do
|
|
property = URI.encode_www_form(auth.property)
|
|
base_url = property_base_url(auth.property)
|
|
|
|
filter_groups =
|
|
if query.filters["page"] do
|
|
[
|
|
%{
|
|
filters: [
|
|
%{
|
|
dimension: "page",
|
|
expression: "https://#{base_url}#{query.filters["page"]}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
end
|
|
|
|
res =
|
|
HTTPoison.post!(
|
|
"https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query",
|
|
Jason.encode!(%{
|
|
startDate: Date.to_iso8601(query.date_range.first),
|
|
endDate: Date.to_iso8601(query.date_range.last),
|
|
dimensions: ["query"],
|
|
rowLimit: limit,
|
|
dimensionFilterGroups: filter_groups || %{}
|
|
}),
|
|
"Content-Type": "application/json",
|
|
Authorization: "Bearer #{auth.access_token}"
|
|
)
|
|
|
|
case res.status_code do
|
|
200 ->
|
|
terms =
|
|
(Jason.decode!(res.body)["rows"] || [])
|
|
|> Enum.filter(fn row -> row["clicks"] > 0 end)
|
|
|> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)
|
|
|
|
{:ok, terms}
|
|
|
|
401 ->
|
|
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body))
|
|
{:error, :invalid_credentials}
|
|
|
|
403 ->
|
|
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body))
|
|
msg = Jason.decode!(res.body)["error"]["message"]
|
|
{:error, msg}
|
|
|
|
_ ->
|
|
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body))
|
|
{:error, :unknown}
|
|
end
|
|
end
|
|
|
|
def get_analytics_view_ids(token) do
|
|
res =
|
|
HTTPoison.get!(
|
|
"https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles",
|
|
Authorization: "Bearer #{token}"
|
|
)
|
|
|
|
case res.status_code do
|
|
200 ->
|
|
profiles =
|
|
Jason.decode!(res.body)
|
|
|> Map.get("items")
|
|
|> Enum.map(fn item ->
|
|
uri = URI.parse(Map.get(item, "websiteUrl", ""))
|
|
|
|
if !uri.host do
|
|
Sentry.capture_message("No URI for view ID", extra: Jason.decode!(res.body))
|
|
end
|
|
|
|
host = uri.host || Map.get(item, "id", "")
|
|
name = Map.get(item, "name")
|
|
{"#{host} - #{name}", Map.get(item, "id")}
|
|
end)
|
|
|> Map.new()
|
|
|
|
{:ok, profiles}
|
|
|
|
_ ->
|
|
Sentry.capture_message("Error fetching Google view ID", extra: Jason.decode!(res.body))
|
|
{:error, res.body}
|
|
end
|
|
end
|
|
|
|
def get_analytics_start_date(view_id, token) do
|
|
report = %{
|
|
viewId: view_id,
|
|
dateRanges: [
|
|
%{
|
|
# The earliest valid date
|
|
startDate: "2005-01-01",
|
|
endDate: Timex.today() |> Date.to_iso8601()
|
|
}
|
|
],
|
|
dimensions: [%{name: "ga:date", histogramBuckets: []}],
|
|
metrics: [%{expression: "ga:pageviews"}],
|
|
hideTotals: true,
|
|
hideValueRanges: true,
|
|
orderBys: [
|
|
%{
|
|
fieldName: "ga:date",
|
|
sortOrder: "ASCENDING"
|
|
}
|
|
],
|
|
pageSize: 1
|
|
}
|
|
|
|
res =
|
|
HTTPoison.post!(
|
|
"https://analyticsreporting.googleapis.com/v4/reports:batchGet",
|
|
Jason.encode!(%{reportRequests: [report]}),
|
|
[Authorization: "Bearer #{token}"],
|
|
timeout: 15_000,
|
|
recv_timeout: 15_000
|
|
)
|
|
|
|
case res.status_code do
|
|
200 ->
|
|
report = List.first(Jason.decode!(res.body)["reports"])
|
|
|
|
date =
|
|
case report["data"]["rows"] do
|
|
[%{"dimensions" => [date_str]}] ->
|
|
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
|
|
|
_ ->
|
|
nil
|
|
end
|
|
|
|
{:ok, date}
|
|
|
|
_ ->
|
|
Sentry.capture_message("Error fetching Google view ID", extra: Jason.decode!(res.body))
|
|
{:error, res.body}
|
|
end
|
|
end
|
|
|
|
# Each element is: {dataset, dimensions, metrics}
|
|
@request_data [
|
|
{
|
|
"imported_visitors",
|
|
["ga:date"],
|
|
[
|
|
"ga:users",
|
|
"ga:pageviews",
|
|
"ga:bounces",
|
|
"ga:sessions",
|
|
"ga:sessionDuration"
|
|
]
|
|
},
|
|
{
|
|
"imported_sources",
|
|
["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"],
|
|
["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
|
},
|
|
{
|
|
"imported_pages",
|
|
["ga:date", "ga:hostname", "ga:pagePath"],
|
|
["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]
|
|
},
|
|
{
|
|
"imported_entry_pages",
|
|
["ga:date", "ga:landingPagePath"],
|
|
["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]
|
|
},
|
|
{
|
|
"imported_exit_pages",
|
|
["ga:date", "ga:exitPagePath"],
|
|
["ga:users", "ga:exits"]
|
|
},
|
|
{
|
|
"imported_locations",
|
|
["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],
|
|
["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
|
},
|
|
{
|
|
"imported_devices",
|
|
["ga:date", "ga:deviceCategory"],
|
|
["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
|
},
|
|
{
|
|
"imported_browsers",
|
|
["ga:date", "ga:browser"],
|
|
["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
|
},
|
|
{
|
|
"imported_operating_systems",
|
|
["ga:date", "ga:operatingSystem"],
|
|
["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
|
}
|
|
]
|
|
|
|
@doc """
|
|
API reference:
|
|
https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest
|
|
|
|
Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer
|
|
"""
|
|
def import_analytics(site, date_range, view_id, access_token) do
|
|
for month_batch <- prepare_batches(date_range, view_id, access_token) do
|
|
tasks =
|
|
for batch_request <- month_batch do
|
|
Task.async(fn -> fetch_and_persist(site, batch_request) end)
|
|
end
|
|
|
|
# 1 hour max to get 1 month's worth of data
|
|
Task.await_many(tasks, 3_600_000)
|
|
end
|
|
|
|
:ok
|
|
end
|
|
|
|
defp prepare_batches(import_date_range, view_id, access_token) do
|
|
total_months = Timex.diff(import_date_range.last, import_date_range.first, :months)
|
|
|
|
monthly_batches =
|
|
for month <- 0..total_months do
|
|
batch_start_date = Timex.shift(import_date_range.first, months: month)
|
|
batch_end_date = Timex.shift(batch_start_date, months: 1, days: -1)
|
|
|
|
batch_end_date =
|
|
if Timex.before?(import_date_range.last, batch_end_date),
|
|
do: import_date_range.last,
|
|
else: batch_end_date
|
|
|
|
Date.range(batch_start_date, batch_end_date)
|
|
end
|
|
|
|
for date_range <- monthly_batches do
|
|
for {dataset, dimensions, metrics} <- @request_data do
|
|
%{
|
|
dataset: dataset,
|
|
dimensions: dimensions,
|
|
metrics: metrics,
|
|
date_range: date_range,
|
|
view_id: view_id,
|
|
access_token: access_token,
|
|
page_token: nil
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
@max_attempts 5
|
|
def fetch_and_persist(site, request, opts \\ []) do
|
|
report_request = build_import_report_request(request)
|
|
http_client = Keyword.get(opts, :http_client, HTTPoison)
|
|
attempt = Keyword.get(opts, :attempt, 1)
|
|
sleep_time = Keyword.get(opts, :sleep_time, 1000)
|
|
|
|
res =
|
|
http_client.post(
|
|
"https://analyticsreporting.googleapis.com/v4/reports:batchGet",
|
|
Jason.encode!(%{reportRequests: [report_request]}),
|
|
[Authorization: "Bearer #{request.access_token}"],
|
|
timeout: 30_000,
|
|
recv_timeout: 30_000
|
|
)
|
|
|
|
with {:ok, %HTTPoison.Response{status_code: 200, body: raw_body}} <- res,
|
|
{:ok, body} <- Jason.decode(raw_body),
|
|
report <- List.first(body["reports"]),
|
|
{:ok, data} <- get_non_empty_rows(report) do
|
|
Imported.from_google_analytics(data, site.id, request.dataset)
|
|
|
|
if report["nextPageToken"] do
|
|
fetch_and_persist(site, %{request | page_token: report["nextPageToken"]})
|
|
else
|
|
:ok
|
|
end
|
|
else
|
|
error ->
|
|
context_key = "request:#{attempt}"
|
|
Sentry.Context.set_extra_context(%{context_key => error})
|
|
|
|
if attempt >= @max_attempts do
|
|
raise "Google API request failed too many times"
|
|
else
|
|
Process.sleep(sleep_time)
|
|
fetch_and_persist(site, request, Keyword.merge(opts, attempt: attempt + 1))
|
|
end
|
|
end
|
|
end
|
|
|
|
defp get_non_empty_rows(report) do
|
|
case get_in(report, ["data", "rows"]) do
|
|
[] -> {:error, :empty_response_rows}
|
|
rows -> {:ok, rows}
|
|
end
|
|
end
|
|
|
|
defp build_import_report_request(request) do
|
|
%{
|
|
viewId: request.view_id,
|
|
dateRanges: [
|
|
%{
|
|
startDate: request.date_range.first,
|
|
endDate: request.date_range.last
|
|
}
|
|
],
|
|
dimensions: Enum.map(request.dimensions, &%{name: &1, histogramBuckets: []}),
|
|
metrics: Enum.map(request.metrics, &%{expression: &1}),
|
|
hideTotals: true,
|
|
hideValueRanges: true,
|
|
orderBys: [
|
|
%{
|
|
fieldName: "ga:date",
|
|
sortOrder: "DESCENDING"
|
|
}
|
|
],
|
|
pageSize: 10_000,
|
|
pageToken: request.page_token
|
|
}
|
|
end
|
|
|
|
defp refresh_if_needed(auth) do
|
|
if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do
|
|
refresh_token(auth)
|
|
else
|
|
{:ok, auth}
|
|
end
|
|
end
|
|
|
|
defp refresh_token(auth) do
|
|
res =
|
|
HTTPoison.post!(
|
|
"https://www.googleapis.com/oauth2/v4/token",
|
|
"client_id=#{client_id()}&client_secret=#{client_secret()}&refresh_token=#{auth.refresh_token}&grant_type=refresh_token&redirect_uri=#{redirect_uri()}",
|
|
"Content-Type": "application/x-www-form-urlencoded"
|
|
)
|
|
|
|
body = Jason.decode!(res.body)
|
|
|
|
if res.status_code == 200 do
|
|
Plausible.Site.GoogleAuth.changeset(auth, %{
|
|
access_token: body["access_token"],
|
|
expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(body["expires_in"])
|
|
})
|
|
|> Plausible.Repo.update()
|
|
else
|
|
{:error, body["error"]}
|
|
end
|
|
end
|
|
|
|
defp client_id() do
|
|
Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
|
|
end
|
|
|
|
defp client_secret() do
|
|
Keyword.fetch!(Application.get_env(:plausible, :google), :client_secret)
|
|
end
|
|
|
|
defp redirect_uri() do
|
|
PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
|
|
end
|
|
end
|