diff --git a/lib/plausible/data_migration/rewrite_funnel_dupes.ex b/lib/plausible/data_migration/rewrite_funnel_dupes.ex new file mode 100644 index 000000000..52db56d34 --- /dev/null +++ b/lib/plausible/data_migration/rewrite_funnel_dupes.ex @@ -0,0 +1,191 @@ +defmodule Plausible.DataMigration.RewriteFunnelDupes do + @moduledoc """ + A data fix migration that seeks funnels having steps + whose goals are equally named. + It then tries to rewrite the duplicate goals using the + oldest goal available. In extreme cases, e.g. when multiple + duplicates are found for a single funnel, it will either + reduce or completely remove the funnel. + This enables us to run a migration later on that will + delete duplicate goals and enforce goal uniqueness at the + database level via a CHECK constraint. + + To run, just call the `run` function. + """ + use Plausible.DataMigration, dir: "FunnelDupeGoals", repo: Plausible.Repo + import Ecto.Query + + def run(_ \\ []) do + {:ok, %{rows: rows}} = run_sql("list-funnels-with-dupe-goal-ids") + data = Enum.map(rows, &to_map/1) + + by_site_id = Enum.group_by(data, & &1.site_id) + + Enum.into(by_site_id, %{}, fn {site_id, data} -> + site_meta = + %{} + |> Map.put(:domain, List.first(data).domain) + |> Map.put(:site_id, site_id) + |> Map.put( + :changes, + Enum.map( + data, + fn d -> + %{ + site_id: site_id, + og_goal_id: d.goal_id, + goal_name: d.goal_name, + funnel_name: d.funnel_name, + funnel_id: d.funnel_id, + funnel_step_id: d.funnel_step_id, + to: Enum.min(d.dupe_goal_ids), + og_steps_count: length(d.all_goal_ids) + } + end + ) + |> Enum.group_by(& &1.funnel_id) + ) + + {site_id, site_meta} + end) + |> translate_to_db_ops() + |> execute() + end + + def execute(data) do + Plausible.Repo.transaction(fn -> + for {site_id, meta} <- data do + IO.puts("\n\nProcessing site ID: #{site_id} (#{meta.domain}):") + + for {funnel_id, changes} <- meta.changes do + IO.puts( + "\nProcessing changes for funnel ID: #{funnel_id} - '#{Enum.at(changes, 0).funnel_name}'" + ) + + Enum.each(changes, fn change -> + apply_change(funnel_id, change) + end) + end + + IO.puts("\nFinished processing site ID: #{site_id} (#{meta.domain}).") + end + end) + end + + def apply_change(funnel_id, %{update_type: :delete_whole_funnel} = change) do + IO.puts( + "Deleting whole funnel '#{change.funnel_name}', there is no way to make it functional without duplicates" + ) + + Plausible.Repo.delete_all( + from(f in Plausible.Funnel, + where: f.site_id == ^change.site_id, + where: f.id == ^funnel_id + ) + ) + end + + def apply_change(funnel_id, %{update_type: :delete_step} = change) do + IO.puts( + "Deleting step ID:#{change.funnel_step_id} '#{change.goal_name}' - otherwise there will be duplicates in the funnel." + ) + + step = + Plausible.Repo.get_by!(Plausible.Funnel.Step, + funnel_id: funnel_id, + id: change.funnel_step_id + ) + + {:ok, _} = Plausible.Repo.delete(step) + end + + def apply_change( + _funnel_id, + %{update_type: :update_step, og_goal_id: goal_id, to: goal_id} = change + ) do + IO.puts( + "Doing nothing for step '#{change.goal_name}' - the duplicate goal exists, but the update is no-op" + ) + + :ok + end + + def apply_change(funnel_id, %{update_type: :update_step} = change) do + IO.puts( + "Updating step '#{change.goal_name}' from goal_id:#{change.og_goal_id} to goal_id:#{change.to}" + ) + + step = + Plausible.Repo.get_by!(Plausible.Funnel.Step, + funnel_id: funnel_id, + id: change.funnel_step_id + ) + + change = Ecto.Changeset.change(step, goal_id: change.to) + {:ok, _} = Plausible.Repo.update(change) + end + + def translate_to_db_ops(data) do + for {site, meta} <- data do + {site, to_db_ops(meta)} + end + end + + def to_db_ops(meta) do + new_changes = + for {funnel_id, changes} <- meta.changes do + safe_step_updates = changes |> Enum.uniq_by(& &1.to) + + steps_to_delete = changes -- safe_step_updates + + safe_step_updates = + safe_step_updates + |> Enum.map(&Map.put(&1, :update_type, :update_step)) + + changes = + Enum.map(steps_to_delete, &Map.put(&1, :update_type, :delete_step)) ++ safe_step_updates + + og_steps_count = Enum.at(changes, 0).og_steps_count + + steps_after_changes = og_steps_count - Enum.count(steps_to_delete) + + if steps_after_changes >= 2 do + {funnel_id, changes} + else + {funnel_id, + [ + Enum.at(changes, 0) + |> Map.put(:update_type, :delete_whole_funnel) + ]} + end + end + + Map.put(meta, :changes, new_changes) + end + + def to_map(row) do + [ + domain, + site_id, + funnel_name, + funnel_id, + funnel_step_id, + goal_id, + goal_name, + dupe_goal_ids, + all_goal_ids + ] = row + + %{ + domain: domain, + site_id: site_id, + funnel_name: funnel_name, + funnel_id: funnel_id, + funnel_step_id: funnel_step_id, + goal_id: goal_id, + goal_name: goal_name, + dupe_goal_ids: dupe_goal_ids, + all_goal_ids: all_goal_ids + } + end +end diff --git a/lib/plausible/funnels.ex b/lib/plausible/funnels.ex index 80c18171f..3a4e82824 100644 --- a/lib/plausible/funnels.ex +++ b/lib/plausible/funnels.ex @@ -45,21 +45,27 @@ defmodule Plausible.Funnels do ] def list(%Plausible.Site{id: site_id}) do Repo.all( - from f in Funnel, + from(f in Funnel, inner_join: steps in assoc(f, :steps), where: f.site_id == ^site_id, select: %{name: f.name, id: f.id, steps_count: count(steps)}, group_by: f.id, order_by: [desc: :id] + ) ) end - @spec delete(Plausible.Site.t(), pos_integer()) :: :ok + @spec delete(Plausible.Site.t() | pos_integer(), pos_integer()) :: :ok def delete(%Plausible.Site{id: site_id}, funnel_id) do + delete(site_id, funnel_id) + end + + def delete(site_id, funnel_id) do Repo.delete_all( - from f in Funnel, + from(f in Funnel, where: f.site_id == ^site_id, where: f.id == ^funnel_id + ) ) :ok @@ -73,7 +79,7 @@ defmodule Plausible.Funnels do def get(site_id, funnel_id) when is_integer(site_id) and is_integer(funnel_id) do q = - from f in Funnel, + from(f in Funnel, where: f.site_id == ^site_id, where: f.id == ^funnel_id, inner_join: steps in assoc(f, :steps), @@ -82,6 +88,7 @@ defmodule Plausible.Funnels do preload: [ steps: {steps, goal: goal} ] + ) Repo.one(q) end diff --git a/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex new file mode 100644 index 000000000..4ea2339c1 --- /dev/null +++ b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex @@ -0,0 +1,62 @@ +WITH DuplicateGoals AS ( + SELECT + g.site_id, + CASE + WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name + WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path + END AS goal_name, + ARRAY_AGG(g.id) AS duplicate_ids + FROM + goals g + WHERE + g.site_id IS NOT NULL + GROUP BY + g.site_id, + goal_name + HAVING COUNT(*) > 1 +), +FunnelStepsWithDuplicateGoals AS ( + SELECT + fs.id, + fs.goal_id, + fs.funnel_id, + f.name, + CASE + WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name + WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path + END AS goal_name, + dg.duplicate_ids + FROM + funnel_steps fs + JOIN goals g ON fs.goal_id = g.id + JOIN funnels f ON fs.funnel_id = f.id + JOIN DuplicateGoals dg ON f.site_id = dg.site_id + WHERE + fs.goal_id = ANY(dg.duplicate_ids) -- Ch +) +SELECT + s.domain, + f.site_id, + f.name as funnel_name, + f.id, + fs.id, + fs.goal_id, + fs.goal_name, + fs.duplicate_ids, + array_agg(fs2.goal_id) +FROM + funnels f +JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id +JOIN funnel_steps fs2 ON f.id = fs2.funnel_id +JOIN sites s ON s.id = f.site_id +GROUP BY + s.domain, + f.site_id, + f.name, + f.id, + fs.id, + fs.goal_id, + fs.goal_name, + fs.duplicate_ids +ORDER BY fs.id ASC +; diff --git a/test/plausible/data_migration/rewrite_funnel_dupes_test.exs b/test/plausible/data_migration/rewrite_funnel_dupes_test.exs new file mode 100644 index 000000000..d9ad24ae0 --- /dev/null +++ b/test/plausible/data_migration/rewrite_funnel_dupes_test.exs @@ -0,0 +1,124 @@ +defmodule Plausible.DataMigration.RewriteFunnelDupesTest do + use Plausible.DataCase, async: true + + alias Plausible.DataMigration.RewriteFunnelDupes + + import ExUnit.CaptureIO + + for goal_type <- ["event_name", "page_path"] do + test "deletes a funnel that cannot be cleaned up from dupe goals (#{goal_type})" do + site = insert(:site) + + goals = + setup_goals!(site, [ + %{unquote(goal_type) => "/AAA"}, + %{unquote(goal_type) => "/AAA"}, + %{unquote(goal_type) => "/AAA"} + ]) + + funnel = setup_funnel!(site, "test", goals) + + io = assert capture_io(fn -> RewriteFunnelDupes.run() end) + assert io =~ "Processing site ID: #{site.id}" + assert io =~ "Deleting whole funnel" + + refute Repo.reload(funnel) + end + + test "reduces a funnel if possible (#{goal_type})" do + site = insert(:site) + + goals = + setup_goals!(site, [ + %{unquote(goal_type) => "/AAA"}, + %{unquote(goal_type) => "/BBB"}, + %{unquote(goal_type) => "/AAA"}, + %{unquote(goal_type) => "/CCC"} + ]) + + funnel = setup_funnel!(site, "test", goals) + + io = assert capture_io(fn -> RewriteFunnelDupes.run() end) + assert io =~ "Processing site ID: #{site.id}" + assert io =~ "Deleting step" + + assert_funnel(site.id, funnel.id, [ + %{unquote(goal_type) => "/AAA"}, + %{unquote(goal_type) => "/BBB"}, + %{unquote(goal_type) => "/CCC"} + ]) + end + end + + test "dupe names in mixed goal types don't matter" do + site = insert(:site) + + gs = [ + %{"event_name" => "/AAA"}, + %{"page_path" => "/AAA"}, + %{"event_name" => "/foo"}, + %{"page_path" => "/foo"} + ] + + goals = setup_goals!(site, gs) + + funnel = setup_funnel!(site, "test", goals) + + capture_io(fn -> RewriteFunnelDupes.run() end) + + assert_funnel(site.id, funnel.id, gs) + end + + test "goals across multiple funnels" do + site = insert(:site) + + goals = [ + %{"event_name" => "/AAA"}, + %{"event_name" => "/AAA"}, + %{"page_path" => "/foo"}, + %{"event_name" => "/AAA"} + ] + + [g1, g2, g3, g4] = setup_goals!(site, goals) + + funnel1 = setup_funnel!(site, "test1", [g1, g2, g3, g4]) + funnel2 = setup_funnel!(site, "test2", [g2, g1, g3, g4]) + funnel3 = setup_funnel!(site, "test3", [g2, g3]) + funnel4 = setup_funnel!(site, "test4", [g2, g1]) + + capture_io(fn -> RewriteFunnelDupes.run() end) + + assert_funnel(site.id, funnel1.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}]) + assert_funnel(site.id, funnel2.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}]) + assert_funnel(site.id, funnel3.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}]) + + refute Plausible.Funnels.get(site.id, funnel4.id) + end + + defp setup_goals!(site, goals) do + for goal <- goals do + {:ok, g} = Plausible.Goals.create(site, goal) + g + end + end + + def setup_funnel!(site, name, goals) do + {:ok, funnel} = Plausible.Funnels.create(site, name, Enum.map(goals, &%{"goal_id" => &1.id})) + funnel + end + + def assert_funnel(site_id, funnel_id, goals) do + funnel_goals = + Plausible.Funnels.get(site_id, funnel_id) + |> Map.fetch!(:steps) + |> Enum.map(& &1.goal) + + assert Enum.count(funnel_goals) == Enum.count(goals) + + Enum.zip(funnel_goals, goals) + |> Enum.each(fn {funnel_goal, expect} -> + key = Map.keys(expect) |> List.first() |> String.to_existing_atom() + assert Map.fetch!(funnel_goal, key) == Map.values(expect) |> List.first() + end) + end +end