Improve import procedure to ensure no time range overlaps (#3970)

* Always scope import ID by site as well

* Do not schedule new import job if there are any site imports in progress

* Disable import buttons when any import is in progress

* Simplify `schedule_job/4` (h/t @RobertJoonas)
master r2024.04.04-428
Adrian Gruntkowski 3 years ago committed by GitHub
parent cffff0340c
commit a7603c9e49
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -62,9 +62,9 @@ defmodule Plausible.Imported do
} }
end end
@spec get_import(non_neg_integer()) :: SiteImport.t() | nil @spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
def get_import(import_id) do def get_import(site, import_id) do
Repo.get(SiteImport, import_id) Repo.get_by(SiteImport, id: import_id, site_id: site.id)
end end
defdelegate listen(), to: Imported.Importer defdelegate listen(), to: Imported.Importer
@ -83,6 +83,16 @@ defmodule Plausible.Imported do
end end
end end
@spec other_imports_in_progress?(SiteImport.t()) :: boolean()
def other_imports_in_progress?(site_import) do
Repo.exists?(
from(i in SiteImport,
where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
)
)
end
defp maybe_filter_by_status(query, nil), do: query defp maybe_filter_by_status(query, nil), do: query
defp maybe_filter_by_status(query, status) do defp maybe_filter_by_status(query, status) do

@ -91,6 +91,7 @@ defmodule Plausible.Imported.Importer do
like LiveView, where notifications can be listened for via `handle_info/2`. like LiveView, where notifications can be listened for via `handle_info/2`.
""" """
alias Plausible.Imported
alias Plausible.Imported.SiteImport alias Plausible.Imported.SiteImport
alias Plausible.Repo alias Plausible.Repo
@ -98,7 +99,7 @@ defmodule Plausible.Imported.Importer do
@callback label() :: String.t() @callback label() :: String.t()
@callback email_template() :: String.t() @callback email_template() :: String.t()
@callback parse_args(map()) :: Keyword.t() @callback parse_args(map()) :: Keyword.t()
@callback import_data(Plausible.Imported.SiteImport.t(), Keyword.t()) :: :ok | {:error, any()} @callback import_data(SiteImport.t(), Keyword.t()) :: :ok | {:error, any()}
@callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()} @callback before_start(SiteImport.t()) :: :ok | {:ok, map()} | {:error, any()}
@callback on_success(SiteImport.t(), map()) :: :ok @callback on_success(SiteImport.t(), map()) :: :ok
@callback on_failure(SiteImport.t()) :: :ok @callback on_failure(SiteImport.t()) :: :ok
@ -108,7 +109,7 @@ defmodule Plausible.Imported.Importer do
@behaviour Plausible.Imported.Importer @behaviour Plausible.Imported.Importer
@spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) :: @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress}
def new_import(site, user, opts) do def new_import(site, user, opts) do
Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1) Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/1)
end end
@ -222,11 +223,15 @@ defmodule Plausible.Imported.Importer do
:ok = listen() :ok = listen()
end end
opts if not Imported.other_imports_in_progress?(site_import) do
|> Keyword.put(:import_id, site_import.id) opts
|> Map.new() |> Keyword.put(:import_id, site_import.id)
|> Plausible.Workers.ImportAnalytics.new() |> Map.new()
|> Oban.insert() |> Plausible.Workers.ImportAnalytics.new()
|> Oban.insert()
else
{:error, :import_in_progress}
end
end end
defp mark_complete(site_import, extra_data, on_success_fun) do defp mark_complete(site_import, extra_data, on_success_fun) do

@ -289,48 +289,48 @@ defmodule PlausibleWeb.GoogleAnalyticsController do
Routes.site_path(conn, :settings_imports_exports, site.domain) Routes.site_path(conn, :settings_imports_exports, site.domain)
end end
case Imported.check_dates(site, start_date, end_date) do import_opts = [
{:ok, start_date, end_date} -> label: property_or_view,
if Google.API.property?(property_or_view) do start_date: start_date,
{:ok, _} = end_date: end_date,
Imported.GoogleAnalytics4.new_import( access_token: access_token,
site, refresh_token: refresh_token,
current_user, token_expires_at: expires_at,
property: property_or_view, legacy: legacy == "true"
label: property_or_view, ]
start_date: start_date,
end_date: end_date, with {:ok, start_date, end_date} <- Imported.check_dates(site, start_date, end_date),
access_token: access_token, import_opts = [{:start_date, start_date}, {:end_date, end_date} | import_opts],
refresh_token: refresh_token, {:ok, _} <- schedule_job(site, current_user, property_or_view, import_opts) do
token_expires_at: expires_at conn
) |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
else |> redirect(external: redirect_route)
{:ok, _} = else
Imported.UniversalAnalytics.new_import( {:error, :no_time_window} ->
site,
current_user,
view_id: property_or_view,
label: property_or_view,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
end
conn conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.") |> put_flash(
:error,
"Import failed. No data could be imported because date range overlaps with existing data."
)
|> redirect(external: redirect_route) |> redirect(external: redirect_route)
{:error, :no_time_window} -> {:error, :import_in_progress} ->
conn conn
|> put_flash( |> put_flash(
:error, :error,
"Import failed. No data could be imported because date range overlaps with existing data." "There's another import still in progress. Please wait until it's completed or cancel it before starting a new one."
) )
|> redirect(external: redirect_route) |> redirect(external: redirect_route)
end end
end end
defp schedule_job(site, current_user, property_or_view, opts) do
if Google.API.property?(property_or_view) do
opts = Keyword.put(opts, :property, property_or_view)
Imported.GoogleAnalytics4.new_import(site, current_user, opts)
else
opts = Keyword.put(opts, :view_id, property_or_view)
Imported.UniversalAnalytics.new_import(site, current_user, opts)
end
end
end end

@ -652,7 +652,7 @@ defmodule PlausibleWeb.SiteController do
|> Plausible.Site.remove_imported_data() |> Plausible.Site.remove_imported_data()
|> Repo.update!() |> Repo.update!()
site_import = Plausible.Imported.get_import(import_id) -> site_import = Plausible.Imported.get_import(site, import_id) ->
Oban.cancel_all_jobs( Oban.cancel_all_jobs(
from(j in Oban.Job, from(j in Oban.Job,
where: where:

@ -42,7 +42,39 @@ defmodule PlausibleWeb.Live.ImportsExportsSettings do
end end
def render(assigns) do def render(assigns) do
import_in_progress? =
Enum.any?(
assigns.site_imports,
&(&1.live_status in [SiteImport.pending(), SiteImport.importing()])
)
assigns = assign(assigns, :import_in_progress?, import_in_progress?)
~H""" ~H"""
<div class="mt-5 flex gap-x-4">
<.button_link
class="w-36 h-20"
theme="bright"
disabled={@import_in_progress?}
href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
>
<img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
</.button_link>
<.button_link
class="w-36 h-20"
theme="bright"
disabled={@import_in_progress?}
href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</.button_link>
</div>
<p :if={@import_in_progress?} class="mt-4 text-red-400 text-sm">
No new imports can be started until the import in progress is completed or cancelled.
</p>
<header class="relative border-b border-gray-200 pb-4"> <header class="relative border-b border-gray-200 pb-4">
<h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100"> <h3 class="mt-8 text-md leading-6 font-medium text-gray-900 dark:text-gray-100">
Existing Imports Existing Imports

@ -10,24 +10,6 @@
<PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" /> <PlausibleWeb.Components.Generic.docs_info slug="google-analytics-import" />
</header> </header>
<div class="mt-5 flex gap-x-4">
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20"
theme="bright"
href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
>
<img src="/images/icon/google_analytics_logo.svg" alt="Google Analytics import" />
</PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20"
theme="bright"
href={"/#{URI.encode_www_form(@site.domain)}/settings/import"}
>
<img class="h-16" src="/images/icon/csv_logo.svg" alt="New CSV import" />
</PlausibleWeb.Components.Generic.button_link>
</div>
<%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings, <%= live_render(@conn, PlausibleWeb.Live.ImportsExportsSettings,
session: %{"domain" => @site.domain} session: %{"domain" => @site.domain}
) %> ) %>

@ -43,7 +43,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
token_expires_at: DateTime.to_iso8601(past) token_expires_at: DateTime.to_iso8601(past)
) )
site_import = Plausible.Imported.get_import(job.args.import_id) site_import = Plausible.Imported.get_import(site, job.args.import_id)
assert site_import.label == "properties/123456" assert site_import.label == "properties/123456"

@ -855,6 +855,46 @@ defmodule PlausibleWeb.GoogleAnalyticsControllerTest do
) )
end end
test "does not start another import when there's any other in progress for the same site", %{
conn: conn,
site: site,
user: user
} do
{:ok, job} =
Plausible.Imported.NoopImporter.new_import(site, user,
start_date: ~D[2020-01-02],
end_date: ~D[2021-10-11]
)
post(conn, "/#{site.domain}/settings/google-import", %{
"property_or_view" => "properties/123456",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => "false"
})
conn =
post(conn, "/#{site.domain}/settings/google-import", %{
"property_or_view" => "123456",
"start_date" => "2018-03-01",
"end_date" => "2022-03-01",
"access_token" => "token",
"refresh_token" => "foo",
"expires_at" => "2022-09-22T20:01:37.112777",
"legacy" => "true"
})
assert Phoenix.Flash.get(conn.assigns.flash, :error) =~
"There's another import still in progress."
assert [%{id: import_id, legacy: false}] = Plausible.Imported.list_all_imports(site)
assert job.args.import_id == import_id
end
for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do for {legacy, view} <- [{"true", :settings_integrations}, {"false", :settings_imports_exports}] do
test "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{ test "redirects to #{view} with no time window error flash error (legacy: #{legacy})", %{
conn: conn, conn: conn,

@ -680,6 +680,25 @@ defmodule PlausibleWeb.SiteControllerTest do
assert resp =~ "Google Analytics (123456)" assert resp =~ "Google Analytics (123456)"
assert resp =~ "(98 page views)" assert resp =~ "(98 page views)"
end end
test "disables import buttons when there's import in progress", %{conn: conn, site: site} do
_site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
_site_import2 = insert(:site_import, site: site, status: SiteImport.importing())
conn = get(conn, "/#{site.domain}/settings/imports-exports")
assert html_response(conn, 200) =~ "No new imports can be started"
end
test "enables import buttons when all imports are in completed or failed state", %{
conn: conn,
site: site
} do
_site_import1 = insert(:site_import, site: site, status: SiteImport.completed())
_site_import2 = insert(:site_import, site: site, status: SiteImport.failed())
conn = get(conn, "/#{site.domain}/settings/imports-exports")
refute html_response(conn, 200) =~ "No new imports can be started"
end
end end
describe "GET /:website/settings/integrations for self-hosting" do describe "GET /:website/settings/integrations for self-hosting" do

Loading…
Cancel
Save