Data Migration: rewrite funnels that contain duplicate goals (#3361)
* Add `RewriteFunnelDupes` data fix To rewrite funnels referencing goals whose names are the same. This enables us to enforce all goals within the site unique later on. * Credo * Update priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com> * Revert unrelated changes * Remove dead code * Update lib/plausible/data_migration/rewrite_funnel_dupes.ex Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com> * Update lib/plausible/data_migration/rewrite_funnel_dupes.ex Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com> --------- Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>pull/3343/head
parent
1a9dd0de90
commit
542385feab
@ -0,0 +1,191 @@
|
||||
defmodule Plausible.DataMigration.RewriteFunnelDupes do
|
||||
@moduledoc """
|
||||
A data fix migration that seeks funnels having steps
|
||||
whose goals are equally named.
|
||||
It then tries to rewrite the duplicate goals using the
|
||||
oldest goal available. In extreme cases, e.g. when multiple
|
||||
duplicates are found for a single funnel, it will either
|
||||
reduce or completely remove the funnel.
|
||||
This enables us to run a migration later on that will
|
||||
delete duplicate goals and enforce goal uniqueness at the
|
||||
database level via a CHECK constraint.
|
||||
|
||||
To run, just call the `run` function.
|
||||
"""
|
||||
use Plausible.DataMigration, dir: "FunnelDupeGoals", repo: Plausible.Repo
|
||||
import Ecto.Query
|
||||
|
||||
def run(_ \\ []) do
|
||||
{:ok, %{rows: rows}} = run_sql("list-funnels-with-dupe-goal-ids")
|
||||
data = Enum.map(rows, &to_map/1)
|
||||
|
||||
by_site_id = Enum.group_by(data, & &1.site_id)
|
||||
|
||||
Enum.into(by_site_id, %{}, fn {site_id, data} ->
|
||||
site_meta =
|
||||
%{}
|
||||
|> Map.put(:domain, List.first(data).domain)
|
||||
|> Map.put(:site_id, site_id)
|
||||
|> Map.put(
|
||||
:changes,
|
||||
Enum.map(
|
||||
data,
|
||||
fn d ->
|
||||
%{
|
||||
site_id: site_id,
|
||||
og_goal_id: d.goal_id,
|
||||
goal_name: d.goal_name,
|
||||
funnel_name: d.funnel_name,
|
||||
funnel_id: d.funnel_id,
|
||||
funnel_step_id: d.funnel_step_id,
|
||||
to: Enum.min(d.dupe_goal_ids),
|
||||
og_steps_count: length(d.all_goal_ids)
|
||||
}
|
||||
end
|
||||
)
|
||||
|> Enum.group_by(& &1.funnel_id)
|
||||
)
|
||||
|
||||
{site_id, site_meta}
|
||||
end)
|
||||
|> translate_to_db_ops()
|
||||
|> execute()
|
||||
end
|
||||
|
||||
def execute(data) do
|
||||
Plausible.Repo.transaction(fn ->
|
||||
for {site_id, meta} <- data do
|
||||
IO.puts("\n\nProcessing site ID: #{site_id} (#{meta.domain}):")
|
||||
|
||||
for {funnel_id, changes} <- meta.changes do
|
||||
IO.puts(
|
||||
"\nProcessing changes for funnel ID: #{funnel_id} - '#{Enum.at(changes, 0).funnel_name}'"
|
||||
)
|
||||
|
||||
Enum.each(changes, fn change ->
|
||||
apply_change(funnel_id, change)
|
||||
end)
|
||||
end
|
||||
|
||||
IO.puts("\nFinished processing site ID: #{site_id} (#{meta.domain}).")
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
def apply_change(funnel_id, %{update_type: :delete_whole_funnel} = change) do
|
||||
IO.puts(
|
||||
"Deleting whole funnel '#{change.funnel_name}', there is no way to make it functional without duplicates"
|
||||
)
|
||||
|
||||
Plausible.Repo.delete_all(
|
||||
from(f in Plausible.Funnel,
|
||||
where: f.site_id == ^change.site_id,
|
||||
where: f.id == ^funnel_id
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def apply_change(funnel_id, %{update_type: :delete_step} = change) do
|
||||
IO.puts(
|
||||
"Deleting step ID:#{change.funnel_step_id} '#{change.goal_name}' - otherwise there will be duplicates in the funnel."
|
||||
)
|
||||
|
||||
step =
|
||||
Plausible.Repo.get_by!(Plausible.Funnel.Step,
|
||||
funnel_id: funnel_id,
|
||||
id: change.funnel_step_id
|
||||
)
|
||||
|
||||
{:ok, _} = Plausible.Repo.delete(step)
|
||||
end
|
||||
|
||||
def apply_change(
|
||||
_funnel_id,
|
||||
%{update_type: :update_step, og_goal_id: goal_id, to: goal_id} = change
|
||||
) do
|
||||
IO.puts(
|
||||
"Doing nothing for step '#{change.goal_name}' - the duplicate goal exists, but the update is no-op"
|
||||
)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
def apply_change(funnel_id, %{update_type: :update_step} = change) do
|
||||
IO.puts(
|
||||
"Updating step '#{change.goal_name}' from goal_id:#{change.og_goal_id} to goal_id:#{change.to}"
|
||||
)
|
||||
|
||||
step =
|
||||
Plausible.Repo.get_by!(Plausible.Funnel.Step,
|
||||
funnel_id: funnel_id,
|
||||
id: change.funnel_step_id
|
||||
)
|
||||
|
||||
change = Ecto.Changeset.change(step, goal_id: change.to)
|
||||
{:ok, _} = Plausible.Repo.update(change)
|
||||
end
|
||||
|
||||
def translate_to_db_ops(data) do
|
||||
for {site, meta} <- data do
|
||||
{site, to_db_ops(meta)}
|
||||
end
|
||||
end
|
||||
|
||||
def to_db_ops(meta) do
|
||||
new_changes =
|
||||
for {funnel_id, changes} <- meta.changes do
|
||||
safe_step_updates = changes |> Enum.uniq_by(& &1.to)
|
||||
|
||||
steps_to_delete = changes -- safe_step_updates
|
||||
|
||||
safe_step_updates =
|
||||
safe_step_updates
|
||||
|> Enum.map(&Map.put(&1, :update_type, :update_step))
|
||||
|
||||
changes =
|
||||
Enum.map(steps_to_delete, &Map.put(&1, :update_type, :delete_step)) ++ safe_step_updates
|
||||
|
||||
og_steps_count = Enum.at(changes, 0).og_steps_count
|
||||
|
||||
steps_after_changes = og_steps_count - Enum.count(steps_to_delete)
|
||||
|
||||
if steps_after_changes >= 2 do
|
||||
{funnel_id, changes}
|
||||
else
|
||||
{funnel_id,
|
||||
[
|
||||
Enum.at(changes, 0)
|
||||
|> Map.put(:update_type, :delete_whole_funnel)
|
||||
]}
|
||||
end
|
||||
end
|
||||
|
||||
Map.put(meta, :changes, new_changes)
|
||||
end
|
||||
|
||||
def to_map(row) do
|
||||
[
|
||||
domain,
|
||||
site_id,
|
||||
funnel_name,
|
||||
funnel_id,
|
||||
funnel_step_id,
|
||||
goal_id,
|
||||
goal_name,
|
||||
dupe_goal_ids,
|
||||
all_goal_ids
|
||||
] = row
|
||||
|
||||
%{
|
||||
domain: domain,
|
||||
site_id: site_id,
|
||||
funnel_name: funnel_name,
|
||||
funnel_id: funnel_id,
|
||||
funnel_step_id: funnel_step_id,
|
||||
goal_id: goal_id,
|
||||
goal_name: goal_name,
|
||||
dupe_goal_ids: dupe_goal_ids,
|
||||
all_goal_ids: all_goal_ids
|
||||
}
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,62 @@
|
||||
WITH DuplicateGoals AS (
|
||||
SELECT
|
||||
g.site_id,
|
||||
CASE
|
||||
WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
|
||||
WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
|
||||
END AS goal_name,
|
||||
ARRAY_AGG(g.id) AS duplicate_ids
|
||||
FROM
|
||||
goals g
|
||||
WHERE
|
||||
g.site_id IS NOT NULL
|
||||
GROUP BY
|
||||
g.site_id,
|
||||
goal_name
|
||||
HAVING COUNT(*) > 1
|
||||
),
|
||||
FunnelStepsWithDuplicateGoals AS (
|
||||
SELECT
|
||||
fs.id,
|
||||
fs.goal_id,
|
||||
fs.funnel_id,
|
||||
f.name,
|
||||
CASE
|
||||
WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
|
||||
WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
|
||||
END AS goal_name,
|
||||
dg.duplicate_ids
|
||||
FROM
|
||||
funnel_steps fs
|
||||
JOIN goals g ON fs.goal_id = g.id
|
||||
JOIN funnels f ON fs.funnel_id = f.id
|
||||
JOIN DuplicateGoals dg ON f.site_id = dg.site_id
|
||||
WHERE
|
||||
fs.goal_id = ANY(dg.duplicate_ids) -- Ch
|
||||
)
|
||||
SELECT
|
||||
s.domain,
|
||||
f.site_id,
|
||||
f.name as funnel_name,
|
||||
f.id,
|
||||
fs.id,
|
||||
fs.goal_id,
|
||||
fs.goal_name,
|
||||
fs.duplicate_ids,
|
||||
array_agg(fs2.goal_id)
|
||||
FROM
|
||||
funnels f
|
||||
JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id
|
||||
JOIN funnel_steps fs2 ON f.id = fs2.funnel_id
|
||||
JOIN sites s ON s.id = f.site_id
|
||||
GROUP BY
|
||||
s.domain,
|
||||
f.site_id,
|
||||
f.name,
|
||||
f.id,
|
||||
fs.id,
|
||||
fs.goal_id,
|
||||
fs.goal_name,
|
||||
fs.duplicate_ids
|
||||
ORDER BY fs.id ASC
|
||||
;
|
||||
@ -0,0 +1,124 @@
|
||||
defmodule Plausible.DataMigration.RewriteFunnelDupesTest do
|
||||
use Plausible.DataCase, async: true
|
||||
|
||||
alias Plausible.DataMigration.RewriteFunnelDupes
|
||||
|
||||
import ExUnit.CaptureIO
|
||||
|
||||
for goal_type <- ["event_name", "page_path"] do
|
||||
test "deletes a funnel that cannot be cleaned up from dupe goals (#{goal_type})" do
|
||||
site = insert(:site)
|
||||
|
||||
goals =
|
||||
setup_goals!(site, [
|
||||
%{unquote(goal_type) => "/AAA"},
|
||||
%{unquote(goal_type) => "/AAA"},
|
||||
%{unquote(goal_type) => "/AAA"}
|
||||
])
|
||||
|
||||
funnel = setup_funnel!(site, "test", goals)
|
||||
|
||||
io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
|
||||
assert io =~ "Processing site ID: #{site.id}"
|
||||
assert io =~ "Deleting whole funnel"
|
||||
|
||||
refute Repo.reload(funnel)
|
||||
end
|
||||
|
||||
test "reduces a funnel if possible (#{goal_type})" do
|
||||
site = insert(:site)
|
||||
|
||||
goals =
|
||||
setup_goals!(site, [
|
||||
%{unquote(goal_type) => "/AAA"},
|
||||
%{unquote(goal_type) => "/BBB"},
|
||||
%{unquote(goal_type) => "/AAA"},
|
||||
%{unquote(goal_type) => "/CCC"}
|
||||
])
|
||||
|
||||
funnel = setup_funnel!(site, "test", goals)
|
||||
|
||||
io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
|
||||
assert io =~ "Processing site ID: #{site.id}"
|
||||
assert io =~ "Deleting step"
|
||||
|
||||
assert_funnel(site.id, funnel.id, [
|
||||
%{unquote(goal_type) => "/AAA"},
|
||||
%{unquote(goal_type) => "/BBB"},
|
||||
%{unquote(goal_type) => "/CCC"}
|
||||
])
|
||||
end
|
||||
end
|
||||
|
||||
test "dupe names in mixed goal types don't matter" do
|
||||
site = insert(:site)
|
||||
|
||||
gs = [
|
||||
%{"event_name" => "/AAA"},
|
||||
%{"page_path" => "/AAA"},
|
||||
%{"event_name" => "/foo"},
|
||||
%{"page_path" => "/foo"}
|
||||
]
|
||||
|
||||
goals = setup_goals!(site, gs)
|
||||
|
||||
funnel = setup_funnel!(site, "test", goals)
|
||||
|
||||
capture_io(fn -> RewriteFunnelDupes.run() end)
|
||||
|
||||
assert_funnel(site.id, funnel.id, gs)
|
||||
end
|
||||
|
||||
test "goals across multiple funnels" do
|
||||
site = insert(:site)
|
||||
|
||||
goals = [
|
||||
%{"event_name" => "/AAA"},
|
||||
%{"event_name" => "/AAA"},
|
||||
%{"page_path" => "/foo"},
|
||||
%{"event_name" => "/AAA"}
|
||||
]
|
||||
|
||||
[g1, g2, g3, g4] = setup_goals!(site, goals)
|
||||
|
||||
funnel1 = setup_funnel!(site, "test1", [g1, g2, g3, g4])
|
||||
funnel2 = setup_funnel!(site, "test2", [g2, g1, g3, g4])
|
||||
funnel3 = setup_funnel!(site, "test3", [g2, g3])
|
||||
funnel4 = setup_funnel!(site, "test4", [g2, g1])
|
||||
|
||||
capture_io(fn -> RewriteFunnelDupes.run() end)
|
||||
|
||||
assert_funnel(site.id, funnel1.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
|
||||
assert_funnel(site.id, funnel2.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
|
||||
assert_funnel(site.id, funnel3.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
|
||||
|
||||
refute Plausible.Funnels.get(site.id, funnel4.id)
|
||||
end
|
||||
|
||||
defp setup_goals!(site, goals) do
|
||||
for goal <- goals do
|
||||
{:ok, g} = Plausible.Goals.create(site, goal)
|
||||
g
|
||||
end
|
||||
end
|
||||
|
||||
def setup_funnel!(site, name, goals) do
|
||||
{:ok, funnel} = Plausible.Funnels.create(site, name, Enum.map(goals, &%{"goal_id" => &1.id}))
|
||||
funnel
|
||||
end
|
||||
|
||||
def assert_funnel(site_id, funnel_id, goals) do
|
||||
funnel_goals =
|
||||
Plausible.Funnels.get(site_id, funnel_id)
|
||||
|> Map.fetch!(:steps)
|
||||
|> Enum.map(& &1.goal)
|
||||
|
||||
assert Enum.count(funnel_goals) == Enum.count(goals)
|
||||
|
||||
Enum.zip(funnel_goals, goals)
|
||||
|> Enum.each(fn {funnel_goal, expect} ->
|
||||
key = Map.keys(expect) |> List.first() |> String.to_existing_atom()
|
||||
assert Map.fetch!(funnel_goal, key) == Map.values(expect) |> List.first()
|
||||
end)
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue