From e1d6d119b08729b61dd8670be232bfc7dc2f7cb8 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 21 Mar 2022 12:47:27 +0200 Subject: [PATCH] Add end date to GA import --- lib/mix/tasks/call_google.ex | 48 +++++++++++++++++++ lib/plausible/google/api.ex | 21 ++------ lib/plausible/stats/clickhouse.ex | 8 ---- .../controllers/site_controller.ex | 24 +++++++--- lib/plausible_web/router.ex | 7 +-- .../site/import_from_google.html.eex | 27 +++++++++++ .../templates/site/settings_general.html.eex | 13 +++-- lib/workers/import_google_analytics.ex | 4 +- .../controllers/site_controller_test.exs | 12 +++-- 9 files changed, 122 insertions(+), 42 deletions(-) create mode 100644 lib/mix/tasks/call_google.ex create mode 100644 lib/plausible_web/templates/site/import_from_google.html.eex diff --git a/lib/mix/tasks/call_google.ex b/lib/mix/tasks/call_google.ex new file mode 100644 index 000000000..32fdad203 --- /dev/null +++ b/lib/mix/tasks/call_google.ex @@ -0,0 +1,48 @@ +defmodule Mix.Tasks.CallGoogle do + use Mix.Task + use Plausible.Repo + require Logger + + @token "ya29.A0ARrdaM-GC-C38oUNmoUlica3v2IGT1eACtvreAFcM4zCYrpQxHdTFfhpmWQfiYAblnKPVRkt_fW-jTTwJ_ScXtkaqq6P_XiBbPIa3g9yJ_F6bP2tEOeHgq07aPlPwHvqXaK8I8gmwtO86VvJLvmhXolHtlXN0g" + @view_id "204615196" + @dimensions ["ga:date"] + @metrics ["ga:users", "ga:pageviews", "ga:bounces", "ga:sessions", "ga:sessionDuration"] + + def run(_) do + Mix.Task.run("app.start") + + report = %{ + viewId: @view_id, + dateRanges: [ + %{ + # The earliest valid date + startDate: "2005-01-01", + endDate: "2019-12-05" + } + ], + dimensions: Enum.map(@dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(@metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true, + orderBys: [ + %{ + fieldName: "ga:date", + sortOrder: "DESCENDING" + } + ], + pageSize: 100_00, + pageToken: "" + } + + Logger.debug(report) + + res = + HTTPoison.post!( + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + Jason.encode!(%{reportRequests: [report]}), + Authorization: "Bearer #{@token}" + ) + + Logger.debug(res.body) + end +end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index be1544bc9..d57d22659 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -153,9 +153,9 @@ defmodule Plausible.Google.Api do end end - def import_analytics(site, profile) do + def import_analytics(site, view_id, end_date) do with {:ok, auth} <- refresh_if_needed(site.google_auth) do - do_import_analytics(site, auth, profile) + do_import_analytics(site, auth, view_id, end_date) end end @@ -165,22 +165,11 @@ defmodule Plausible.Google.Api do Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer """ - def do_import_analytics(site, auth, profile) do - end_date = - Plausible.Stats.Clickhouse.pageviews_begin(site) - |> NaiveDateTime.to_date() - - end_date = - if end_date == ~D[1970-01-01] do - Timex.today() - else - end_date - end - + def do_import_analytics(site, auth, view_id, end_date) do request = %{ auth: auth, - profile: profile, - end_date: Date.to_iso8601(end_date) + profile: view_id, + end_date: end_date } # Each element is: {dataset, dimensions, metrics} diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 76dd9eb70..545f66ce4 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -560,12 +560,4 @@ defmodule Plausible.Stats.Clickhouse do db_query end end - - def pageviews_begin(site) do - ClickhouseRepo.one( - from e in "events", - where: e.domain == ^site.domain and e.name == "pageview", - select: min(e.timestamp) - ) - end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 7ec776fdd..c53cabb48 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -643,7 +643,21 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings/general") end - def import_from_google(conn, %{"profile" => profile}) do + def import_from_google_form(conn, _params) do + site = conn.assigns[:site] |> Repo.preload(:google_auth) + + view_ids = Plausible.Google.Api.get_analytics_view_ids(site) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("import_from_google.html", + site: site, + view_ids: view_ids, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import_from_google(conn, %{"view_id" => view_id, "end_date" => end_date}) do site = conn.assigns[:site] |> Repo.preload(:google_auth) @@ -654,16 +668,12 @@ defmodule PlausibleWeb.SiteController do |> put_flash(:error, "Data already imported from: #{site.imported_data.source}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) - profile == "" -> - conn - |> put_flash(:error, "A Google Analytics profile must be selected") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) - true -> job = Plausible.Workers.ImportGoogleAnalytics.new(%{ "site_id" => site.id, - "profile" => profile + "view_id" => view_id, + "end_date" => end_date }) Ecto.Multi.new() diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index c118d053d..1ef1e9b25 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -238,10 +238,11 @@ defmodule PlausibleWeb.Router do delete "/:website", SiteController, :delete_site delete "/:website/stats", SiteController, :reset_stats - get "/:domain/export", StatsController, :csv_export - get "/:domain/*path", StatsController, :stats - + get "/:website/import/google-analytics", SiteController, :import_from_google_form post "/:website/settings/google-import", SiteController, :import_from_google delete "/:website/settings/forget-imported", SiteController, :forget_imported + + get "/:domain/export", StatsController, :csv_export + get "/:domain/*path", StatsController, :stats end end diff --git a/lib/plausible_web/templates/site/import_from_google.html.eex b/lib/plausible_web/templates/site/import_from_google.html.eex new file mode 100644 index 000000000..441670302 --- /dev/null +++ b/lib/plausible_web/templates/site/import_from_google.html.eex @@ -0,0 +1,27 @@ +<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +

Import from Google Analytics

+ + <%= case @view_ids do %> + <% {:ok, view_ids} -> %> + +
+ You can choose the view ID that will be imported to the <%= @site.domain %> dashboard. +
+ +
+ <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %> + <%= select f, :view_id, view_ids, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> +
+ +
+ <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %> +
+ <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %> +
+
+ <% {:error, error} -> %> +

The following error occurred when fetching your Google Analytics view ids.

+

<%= error %>

+ <% end %> + <%= submit "Import", class: "button" %> +<% end %> diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index d0aed180d..6daad6dd2 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -86,15 +86,22 @@ <%= case @google_profiles do %> <% {:ok, profiles} -> %>

- Select the Google Analytics profile you would like to import data from. + Select the Google Analytics view id you would like to import data from.

<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %>
-
- <%= select f, :profile, profiles, prompt: "(Choose profile)", class: "dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" %> + <%= label f, :view_id, "Google Analytics view id", class: "block text-sm font-medium text-gray-700" %> + <%= select f, :view_id, profiles, prompt: "(Choose view id)", required: "true", class: "mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md" %> +
+ +
+ <%= label f, :end_date, "Import end date", class: "block text-sm font-medium text-gray-700" %> +
+ <%= date_input f, :end_date, value: Timex.today() |> Date.to_iso8601(), required: "true", class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>
+ <%= submit "Import", class: "button" %> <% end %> diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index 0e32b8c10..1387018a9 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -8,14 +8,14 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do @impl Oban.Worker def perform( - %Oban.Job{args: %{"site_id" => site_id, "profile" => profile}}, + %Oban.Job{args: %{"site_id" => site_id, "view_id" => view_id, "end_date" => end_date}}, google_api \\ Plausible.Google.Api ) do site = Repo.get(Plausible.Site, site_id) |> Repo.preload([:google_auth, [memberships: :user]]) - case google_api.import_analytics(site, profile) do + case google_api.import_analytics(site, view_id, end_date) do {:ok, _} -> Plausible.Site.import_success(site) |> Repo.update!() diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index eace504ee..1c7903838 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -725,7 +725,10 @@ defmodule PlausibleWeb.SiteControllerTest do setup [:create_user, :log_in, :create_new_site] test "adds in-progress imported tag to site", %{conn: conn, site: site} do - post(conn, "/#{site.domain}/settings/google-import", %{"profile" => "123"}) + post(conn, "/#{site.domain}/settings/google-import", %{ + "profile" => "123", + "end_date" => "2022-03-01" + }) imported_data = Repo.reload(site).imported_data @@ -736,11 +739,14 @@ defmodule PlausibleWeb.SiteControllerTest do end test "schedules an import job in Oban", %{conn: conn, site: site} do - post(conn, "/#{site.domain}/settings/google-import", %{"profile" => "123"}) + post(conn, "/#{site.domain}/settings/google-import", %{ + "view_id" => "123", + "end_date" => "2022-03-01" + }) assert_enqueued( worker: Plausible.Workers.ImportGoogleAnalytics, - args: %{"site_id" => site.id, "profile" => "123"} + args: %{"site_id" => site.id, "view_id" => "123", "end_date" => "2022-03-01"} ) end end