Google Analytics Import Refactor (#2046)
* Create separate module for GA HTTP requests * Fetch GA data entirely instead of monthly * Add buffering to GA imports * Change positional args to maps when serializing from GA * Create Google Analytics VCR testspull/2082/head
parent
2d7dee7067
commit
4b9032d822
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@ -0,0 +1,96 @@
|
||||
defmodule Plausible.Google.Buffer do
|
||||
@moduledoc """
|
||||
This GenServer inserts records into Clickhouse `imported_*` tables. Multiple buffers are
|
||||
automatically created for each table. Records are flushed when the table buffer reaches the
|
||||
maximum size, defined by `max_buffer_size/0`.
|
||||
"""
|
||||
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
def start_link do
|
||||
GenServer.start_link(__MODULE__, nil)
|
||||
end
|
||||
|
||||
def init(_opts) do
|
||||
{:ok, %{buffers: %{}}}
|
||||
end
|
||||
|
||||
@spec insert_many(pid(), term(), [map()]) :: :ok
|
||||
@doc """
|
||||
Puts the given records into the table buffer.
|
||||
"""
|
||||
def insert_many(pid, table_name, records) do
|
||||
GenServer.call(pid, {:insert_many, table_name, records})
|
||||
end
|
||||
|
||||
@spec size(pid(), term()) :: non_neg_integer()
|
||||
@doc """
|
||||
Returns the total count of items in the given table buffer.
|
||||
"""
|
||||
def size(pid, table_name) do
|
||||
GenServer.call(pid, {:get_size, table_name})
|
||||
end
|
||||
|
||||
@spec flush(pid()) :: :ok
|
||||
@doc """
|
||||
Flushes all table buffers to Clickhouse.
|
||||
"""
|
||||
def flush(pid, timeout \\ :infinity) do
|
||||
GenServer.call(pid, :flush_all_buffers, timeout)
|
||||
end
|
||||
|
||||
def stop(pid) do
|
||||
GenServer.stop(pid)
|
||||
end
|
||||
|
||||
def handle_call({:get_size, table_name}, _from, %{buffers: buffers} = state) do
|
||||
size =
|
||||
buffers
|
||||
|> Map.get(table_name, [])
|
||||
|> length()
|
||||
|
||||
{:reply, size, state}
|
||||
end
|
||||
|
||||
def handle_call({:insert_many, table_name, records}, _from, %{buffers: buffers} = state) do
|
||||
Logger.info("Import: Adding #{length(records)} to #{table_name} buffer")
|
||||
|
||||
new_buffer = Map.get(buffers, table_name, []) ++ records
|
||||
new_state = put_in(state.buffers[table_name], new_buffer)
|
||||
|
||||
if length(new_buffer) >= max_buffer_size() do
|
||||
{:reply, :ok, new_state, {:continue, {:flush, table_name}}}
|
||||
else
|
||||
{:reply, :ok, new_state}
|
||||
end
|
||||
end
|
||||
|
||||
def handle_call(:flush_all_buffers, _from, state) do
|
||||
Enum.each(state.buffers, fn {table_name, records} ->
|
||||
flush_buffer(records, table_name)
|
||||
end)
|
||||
|
||||
{:reply, :ok, put_in(state.buffers, %{})}
|
||||
end
|
||||
|
||||
def handle_continue({:flush, table_name}, state) do
|
||||
flush_buffer(state.buffers[table_name], table_name)
|
||||
{:noreply, put_in(state.buffers[table_name], [])}
|
||||
end
|
||||
|
||||
defp max_buffer_size do
|
||||
:plausible
|
||||
|> Application.get_env(:google)
|
||||
|> Keyword.fetch!(:max_buffer_size)
|
||||
end
|
||||
|
||||
defp flush_buffer(records, table_name) do
|
||||
# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
|
||||
# sleep call slows down the flushing
|
||||
Process.sleep(1000)
|
||||
|
||||
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
|
||||
Plausible.ClickhouseRepo.insert_all(table_name, records)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,208 @@
|
||||
defmodule Plausible.Google.HTTP do
|
||||
@spec get_report(module(), Plausible.Google.ReportRequest.t()) ::
|
||||
{:ok, {[map()], String.t() | nil}} | {:error, any()}
|
||||
def get_report(http_client, %Plausible.Google.ReportRequest{} = report_request) do
|
||||
params =
|
||||
Jason.encode!(%{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: report_request.view_id,
|
||||
dateRanges: [
|
||||
%{
|
||||
startDate: report_request.date_range.first,
|
||||
endDate: report_request.date_range.last
|
||||
}
|
||||
],
|
||||
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
|
||||
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
|
||||
pageSize: report_request.page_size,
|
||||
pageToken: report_request.page_token
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
response =
|
||||
:post
|
||||
|> Finch.build(
|
||||
"https://analyticsreporting.googleapis.com/v4/reports:batchGet",
|
||||
[{"Authorization", "Bearer #{report_request.access_token}"}],
|
||||
params
|
||||
)
|
||||
|> http_client.request(Plausible.Finch)
|
||||
|
||||
with {:ok, %{status: 200, body: body}} <- response,
|
||||
{:ok, %{"reports" => [report | _]}} <- Jason.decode(body),
|
||||
token <- Map.get(report, "nextPageToken"),
|
||||
report <- convert_to_maps(report) do
|
||||
{:ok, {report, token}}
|
||||
end
|
||||
end
|
||||
|
||||
defp convert_to_maps(%{
|
||||
"data" => %{"rows" => rows},
|
||||
"columnHeader" => %{
|
||||
"dimensions" => dimension_headers,
|
||||
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
|
||||
}
|
||||
}) do
|
||||
metric_headers = Enum.map(metric_headers, & &1["name"])
|
||||
|
||||
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
|
||||
metrics = Enum.zip(metric_headers, metrics)
|
||||
dimensions = Enum.zip(dimension_headers, dimensions)
|
||||
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
|
||||
end)
|
||||
end
|
||||
|
||||
def list_sites(access_token) do
|
||||
"https://www.googleapis.com/webmasters/v3/sites"
|
||||
|> HTTPoison.get!("Content-Type": "application/json", Authorization: "Bearer #{access_token}")
|
||||
|> Map.get(:body)
|
||||
|> Jason.decode!()
|
||||
|> then(&{:ok, &1})
|
||||
end
|
||||
|
||||
def fetch_access_token(code) do
|
||||
"https://www.googleapis.com/oauth2/v4/token"
|
||||
|> HTTPoison.post!(
|
||||
"client_id=#{client_id()}&client_secret=#{client_secret()}&code=#{code}&grant_type=authorization_code&redirect_uri=#{redirect_uri()}",
|
||||
"Content-Type": "application/x-www-form-urlencoded"
|
||||
)
|
||||
|> Map.get(:body)
|
||||
|> Jason.decode!()
|
||||
end
|
||||
|
||||
def list_views_for_user(access_token) do
|
||||
"https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
|
||||
|> HTTPoison.get!(Authorization: "Bearer #{access_token}")
|
||||
|> case do
|
||||
%{body: body, status_code: 200} ->
|
||||
{:ok, Jason.decode!(body)}
|
||||
|
||||
%{body: body} ->
|
||||
Sentry.capture_message("Error fetching Google view ID", extra: Jason.decode!(body))
|
||||
{:error, body}
|
||||
end
|
||||
end
|
||||
|
||||
def list_stats(access_token, property, date_range, limit, page \\ nil) do
|
||||
property = URI.encode_www_form(property)
|
||||
|
||||
filter_groups =
|
||||
if page do
|
||||
url = property_base_url(property)
|
||||
[%{filters: [%{dimension: "page", expression: "https://#{url}#{page}"}]}]
|
||||
else
|
||||
%{}
|
||||
end
|
||||
|
||||
params =
|
||||
Jason.encode!(%{
|
||||
startDate: Date.to_iso8601(date_range.first),
|
||||
endDate: Date.to_iso8601(date_range.last),
|
||||
dimensions: ["query"],
|
||||
rowLimit: limit,
|
||||
dimensionFilterGroups: filter_groups
|
||||
})
|
||||
|
||||
"https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query"
|
||||
|> HTTPoison.post!(params,
|
||||
"Content-Type": "application/json",
|
||||
Authorization: "Bearer #{access_token}"
|
||||
)
|
||||
|> case do
|
||||
%{status_code: 200, body: body} ->
|
||||
{:ok, Jason.decode!(body)}
|
||||
|
||||
%{status_code: 401, body: body} ->
|
||||
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body))
|
||||
{:error, :invalid_credentials}
|
||||
|
||||
%{status_code: 403, body: body} ->
|
||||
body = Jason.decode!(body)
|
||||
Sentry.capture_message("Error fetching Google queries", extra: body)
|
||||
{:error, get_in(body, ["error", "message"])}
|
||||
|
||||
%{body: body} ->
|
||||
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body))
|
||||
{:error, :unknown}
|
||||
end
|
||||
end
|
||||
|
||||
defp property_base_url("sc-domain:" <> domain), do: "https://" <> domain
|
||||
defp property_base_url(url), do: url
|
||||
|
||||
def refresh_auth_token(refresh_token) do
|
||||
"https://www.googleapis.com/oauth2/v4/token"
|
||||
|> HTTPoison.post!(
|
||||
"client_id=#{client_id()}&client_secret=#{client_secret()}&refresh_token=#{refresh_token}&grant_type=refresh_token&redirect_uri=#{redirect_uri()}",
|
||||
"Content-Type": "application/x-www-form-urlencoded"
|
||||
)
|
||||
|> case do
|
||||
%{body: body, status_code: 200} ->
|
||||
{:ok, Jason.decode!(body)}
|
||||
|
||||
%{body: body} ->
|
||||
body
|
||||
|> Jason.decode!(body)
|
||||
|> Map.get("error")
|
||||
|> then(&{:error, &1})
|
||||
end
|
||||
end
|
||||
|
||||
@earliest_valid_date "2005-01-01"
|
||||
def get_analytics_start_date(view_id, access_token) do
|
||||
params =
|
||||
Jason.encode!(%{
|
||||
reportRequests: [
|
||||
%{
|
||||
viewId: view_id,
|
||||
dateRanges: [
|
||||
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
|
||||
],
|
||||
dimensions: [%{name: "ga:date", histogramBuckets: []}],
|
||||
metrics: [%{expression: "ga:pageviews"}],
|
||||
hideTotals: true,
|
||||
hideValueRanges: true,
|
||||
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
|
||||
pageSize: 1
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
"https://analyticsreporting.googleapis.com/v4/reports:batchGet"
|
||||
|> HTTPoison.post!(
|
||||
params,
|
||||
[Authorization: "Bearer #{access_token}"],
|
||||
timeout: 15_000,
|
||||
recv_timeout: 15_000
|
||||
)
|
||||
|> case do
|
||||
%{status_code: 200, body: body} ->
|
||||
report = List.first(Jason.decode!(body)["reports"])
|
||||
|
||||
date =
|
||||
case report["data"]["rows"] do
|
||||
[%{"dimensions" => [date_str]}] ->
|
||||
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
{:ok, date}
|
||||
|
||||
%{body: body} ->
|
||||
Sentry.capture_message("Error fetching Google view ID", extra: Jason.decode!(body))
|
||||
{:error, body}
|
||||
end
|
||||
end
|
||||
|
||||
defp config, do: Application.get_env(:plausible, :google)
|
||||
defp client_id, do: Keyword.fetch!(config(), :client_id)
|
||||
defp client_secret, do: Keyword.fetch!(config(), :client_secret)
|
||||
defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
|
||||
end
|
||||
@ -0,0 +1,80 @@
|
||||
defmodule Plausible.Google.ReportRequest do
|
||||
defstruct [
|
||||
:dataset,
|
||||
:dimensions,
|
||||
:metrics,
|
||||
:date_range,
|
||||
:view_id,
|
||||
:access_token,
|
||||
:page_token,
|
||||
:page_size
|
||||
]
|
||||
|
||||
@type t() :: %__MODULE__{
|
||||
dataset: String.t(),
|
||||
dimensions: [String.t()],
|
||||
metrics: [String.t()],
|
||||
date_range: Date.Range.t(),
|
||||
view_id: term(),
|
||||
access_token: String.t(),
|
||||
page_token: String.t() | nil,
|
||||
page_size: non_neg_integer()
|
||||
}
|
||||
|
||||
def full_report do
|
||||
[
|
||||
%__MODULE__{
|
||||
dataset: "imported_visitors",
|
||||
dimensions: ["ga:date"],
|
||||
metrics: ["ga:users", "ga:pageviews", "ga:bounces", "ga:sessions", "ga:sessionDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_sources",
|
||||
dimensions: [
|
||||
"ga:date",
|
||||
"ga:source",
|
||||
"ga:medium",
|
||||
"ga:campaign",
|
||||
"ga:adContent",
|
||||
"ga:keyword"
|
||||
],
|
||||
metrics: ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_pages",
|
||||
dimensions: ["ga:date", "ga:hostname", "ga:pagePath"],
|
||||
metrics: ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_entry_pages",
|
||||
dimensions: ["ga:date", "ga:landingPagePath"],
|
||||
metrics: ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_exit_pages",
|
||||
dimensions: ["ga:date", "ga:exitPagePath"],
|
||||
metrics: ["ga:users", "ga:exits"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_locations",
|
||||
dimensions: ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"],
|
||||
metrics: ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_devices",
|
||||
dimensions: ["ga:date", "ga:deviceCategory"],
|
||||
metrics: ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_browsers",
|
||||
dimensions: ["ga:date", "ga:browser"],
|
||||
metrics: ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
||||
},
|
||||
%__MODULE__{
|
||||
dataset: "imported_operating_systems",
|
||||
dimensions: ["ga:date", "ga:operatingSystem"],
|
||||
metrics: ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"]
|
||||
}
|
||||
]
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,96 @@
|
||||
defmodule Plausible.Google.BufferTest do
|
||||
use Plausible.DataCase, async: true
|
||||
import Plausible.TestUtils
|
||||
import Ecto.Query
|
||||
alias Plausible.Google.Buffer
|
||||
|
||||
setup [:create_user, :create_new_site, :set_buffer_size]
|
||||
|
||||
defp set_buffer_size(_setup_args) do
|
||||
Application.put_env(:plausible, :google, max_buffer_size: 10)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp imported_count(%{id: site_id}, table_name) do
|
||||
table_name
|
||||
|> from()
|
||||
|> where([record], record.site_id == ^site_id)
|
||||
|> Plausible.ClickhouseRepo.aggregate(:count)
|
||||
end
|
||||
|
||||
defp build_records(count, factory_name, site) do
|
||||
count
|
||||
|> build_list(factory_name, site_id: site.id)
|
||||
|> Enum.map(&Map.drop(&1, [:table]))
|
||||
end
|
||||
|
||||
test "insert_many/3 flushes when buffer reaches limit", %{site: site} do
|
||||
{:ok, pid} = Buffer.start_link()
|
||||
|
||||
imported_visitors = build_records(9, :imported_visitors, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
assert Buffer.size(pid, "imported_visitors") == 9
|
||||
assert imported_count(site, "imported_visitors") == 0, "expected not to have flushed"
|
||||
|
||||
imported_visitors = build_records(1, :imported_visitors, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
assert Buffer.size(pid, "imported_visitors") == 0
|
||||
assert imported_count(site, "imported_visitors") == 10, "expected to have flushed"
|
||||
end
|
||||
|
||||
test "insert_many/3 uses separate buffers for each table", %{site: site} do
|
||||
{:ok, pid} = Buffer.start_link()
|
||||
|
||||
imported_visitors = build_records(9, :imported_visitors, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
assert Buffer.size(pid, "imported_visitors") == 9
|
||||
assert imported_count(site, "imported_visitors") == 0, "expected not to have flushed"
|
||||
|
||||
imported_sources = build_records(1, :imported_sources, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_sources", imported_sources)
|
||||
assert Buffer.size(pid, "imported_sources") == 1
|
||||
assert imported_count(site, "imported_visitors") == 0, "expected not to have flushed"
|
||||
|
||||
imported_visitors = build_records(1, :imported_visitors, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
assert Buffer.size(pid, "imported_visitors") == 0
|
||||
assert imported_count(site, "imported_visitors") == 10, "expected to have flushed"
|
||||
|
||||
imported_sources = build_records(9, :imported_sources, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_sources", imported_sources)
|
||||
assert Buffer.size(pid, "imported_sources") == 0
|
||||
assert imported_count(site, "imported_sources") == 10, "expected to have flushed"
|
||||
end
|
||||
|
||||
test "insert_many/3 flushes buffer automatically with many records", %{site: site} do
|
||||
{:ok, pid} = Buffer.start_link()
|
||||
|
||||
imported_visitors = build_records(50, :imported_visitors, site)
|
||||
assert :ok == Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
assert Buffer.size(pid, "imported_visitors") == 0
|
||||
assert imported_count(site, "imported_visitors") == 50, "expected to have flushed"
|
||||
end
|
||||
|
||||
test "flush/2 flushes all buffers", %{site: site} do
|
||||
{:ok, pid} = Buffer.start_link()
|
||||
|
||||
imported_sources = build_records(1, :imported_sources, site)
|
||||
Buffer.insert_many(pid, "imported_sources", imported_sources)
|
||||
|
||||
imported_visitors = build_records(1, :imported_visitors, site)
|
||||
Buffer.insert_many(pid, "imported_visitors", imported_visitors)
|
||||
|
||||
imported_operating_systems = build_records(2, :imported_operating_systems, site)
|
||||
Buffer.insert_many(pid, "imported_operating_systems", imported_operating_systems)
|
||||
|
||||
assert :ok == Buffer.flush(pid, :timer.seconds(4))
|
||||
|
||||
assert Buffer.size(pid, "imported_sources") == 0
|
||||
assert Buffer.size(pid, "imported_visitors") == 0
|
||||
assert Buffer.size(pid, "imported_operating_systems") == 0
|
||||
|
||||
assert imported_count(site, "imported_sources") == 1
|
||||
assert imported_count(site, "imported_visitors") == 1
|
||||
assert imported_count(site, "imported_operating_systems") == 2
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,39 @@
|
||||
defmodule Plausible.Google.Api.VCRTest do
|
||||
use Plausible.DataCase, async: false
|
||||
use ExVCR.Mock, adapter: ExVCR.Adapter.Finch
|
||||
require Ecto.Query
|
||||
import Plausible.TestUtils
|
||||
|
||||
setup [:create_user, :create_site]
|
||||
|
||||
defp get_insert_count do
|
||||
Plausible.ClickhouseRepo.aggregate(
|
||||
from(ql in "query_log",
|
||||
prefix: "system",
|
||||
where: ql.query_kind == "Insert" and ql.is_initial_query == true
|
||||
),
|
||||
:count
|
||||
)
|
||||
end
|
||||
|
||||
test "imports page views from Google Analytics", %{site: site} do
|
||||
use_cassette "google_analytics_import#1", match_requests_on: [:request_body] do
|
||||
inserts_before_importing = get_insert_count()
|
||||
before_importing_timestamp = DateTime.utc_now()
|
||||
|
||||
access_token = "***"
|
||||
view_id = "54297898"
|
||||
date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])
|
||||
|
||||
assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token)
|
||||
|
||||
total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second)
|
||||
total_inserts = get_insert_count() - inserts_before_importing
|
||||
|
||||
assert total_inserts / total_seconds > 1.0,
|
||||
"should not call Clickhouse more than 1 time per second"
|
||||
|
||||
assert 1_495_150 == Plausible.Stats.Clickhouse.imported_pageview_count(site)
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue