From 33eed9d7db2bb1f4e6bcfccdb63bf8b7fe83574e Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 4 Apr 2024 18:55:14 +0200 Subject: [PATCH] Delete imports which have no stats (#3972) --- lib/plausible/data_migration/site_imports.ex | 69 ++++++++++++------- .../data_migration/site_imports_test.exs | 16 +++++ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.ex index 5c84b642a..fc4c91b2d 100644 --- a/lib/plausible/data_migration/site_imports.ex +++ b/lib/plausible/data_migration/site_imports.ex @@ -72,35 +72,36 @@ defmodule Plausible.DataMigration.SiteImports do end_date = imported_stats_end_date(site.id, import_ids) - end_date = - if !end_date do - IO.puts( - "Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Setting end date to minimum." - ) + if !end_date do + IO.puts( + "Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it." + ) - Date.add(site_import.start_date, 2) - else - end_date - end - - end_date = - if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do - end_date - else - IO.puts( - "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping." - ) - - site_import.end_date + if site_import.legacy do + clear_imported_data(site, dry_run?) end - site_import - |> Ecto.Changeset.change(end_date: end_date) - |> update!(dry_run?) - - IO.puts( - "End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}" - ) + delete!(site_import, dry_run?) + else + end_date = + if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do + end_date + else + IO.puts( + "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping." + ) + + site_import.end_date + end + + site_import + |> Ecto.Changeset.change(end_date: end_date) + |> update!(dry_run?) + + IO.puts( + "End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}" + ) + end end IO.puts("Done processing site ID #{site.id}") @@ -149,6 +150,14 @@ defmodule Plausible.DataMigration.SiteImports do end end + defp clear_imported_data(site, false = _dry_run?) do + Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil]) + end + + defp clear_imported_data(site, true = _dry_run?) do + %{site | imported_data: nil} + end + defp update!(changeset, false = _dry_run?) do Repo.update!(changeset) end @@ -161,6 +170,14 @@ defmodule Plausible.DataMigration.SiteImports do end end + defp delete!(entity, false = _dry_run?) do + Repo.delete!(entity) + end + + defp delete!(entity, true = _dry_run?) do + entity + end + defp max_date_query(schema, site_id, import_ids) do from(q in schema, where: q.site_id == ^site_id, diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exs index c8e48d0d3..a99ded097 100644 --- a/test/plausible/data_migration/site_imports_test.exs +++ b/test/plausible/data_migration/site_imports_test.exs @@ -91,6 +91,22 @@ defmodule Plausible.DataMigration.SiteImportsTest do assert site_import.source == :universal_analytics end + test "removes site import when there are no stats" do + site = + insert(:site) + |> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok") + |> Repo.update!() + + _another_site_import = insert(:site_import, site: site) + + assert capture_io(fn -> + assert :ok = SiteImports.run(dry_run?: false) + end) =~ "Processing 1 site" + + site = Repo.reload!(site) + assert [] = Imported.list_all_imports(site) + end + test "leaves site and imports unchanged if everything fits" do site = insert(:site)