Data Migration: rewrite funnels that contain duplicate goals (#3361)

* Add `RewriteFunnelDupes` data fix

To rewrite funnels referencing goals whose names
are the same. This enables us to enforce all goals
within the site unique later on.

* Credo

* Update priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>

* Revert unrelated changes

* Remove dead code

* Update lib/plausible/data_migration/rewrite_funnel_dupes.ex

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>

* Update lib/plausible/data_migration/rewrite_funnel_dupes.ex

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>

---------

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
pull/3343/head
hq1 3 years ago committed by GitHub
parent 1a9dd0de90
commit 542385feab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,191 @@
defmodule Plausible.DataMigration.RewriteFunnelDupes do
@moduledoc """
A data fix migration that seeks funnels having steps
whose goals are equally named.
It then tries to rewrite the duplicate goals using the
oldest goal available. In extreme cases, e.g. when multiple
duplicates are found for a single funnel, it will either
reduce or completely remove the funnel.
This enables us to run a migration later on that will
delete duplicate goals and enforce goal uniqueness at the
database level via a CHECK constraint.
To run, just call the `run` function.
"""
use Plausible.DataMigration, dir: "FunnelDupeGoals", repo: Plausible.Repo
import Ecto.Query
def run(_ \\ []) do
{:ok, %{rows: rows}} = run_sql("list-funnels-with-dupe-goal-ids")
data = Enum.map(rows, &to_map/1)
by_site_id = Enum.group_by(data, & &1.site_id)
Enum.into(by_site_id, %{}, fn {site_id, data} ->
site_meta =
%{}
|> Map.put(:domain, List.first(data).domain)
|> Map.put(:site_id, site_id)
|> Map.put(
:changes,
Enum.map(
data,
fn d ->
%{
site_id: site_id,
og_goal_id: d.goal_id,
goal_name: d.goal_name,
funnel_name: d.funnel_name,
funnel_id: d.funnel_id,
funnel_step_id: d.funnel_step_id,
to: Enum.min(d.dupe_goal_ids),
og_steps_count: length(d.all_goal_ids)
}
end
)
|> Enum.group_by(& &1.funnel_id)
)
{site_id, site_meta}
end)
|> translate_to_db_ops()
|> execute()
end
def execute(data) do
Plausible.Repo.transaction(fn ->
for {site_id, meta} <- data do
IO.puts("\n\nProcessing site ID: #{site_id} (#{meta.domain}):")
for {funnel_id, changes} <- meta.changes do
IO.puts(
"\nProcessing changes for funnel ID: #{funnel_id} - '#{Enum.at(changes, 0).funnel_name}'"
)
Enum.each(changes, fn change ->
apply_change(funnel_id, change)
end)
end
IO.puts("\nFinished processing site ID: #{site_id} (#{meta.domain}).")
end
end)
end
def apply_change(funnel_id, %{update_type: :delete_whole_funnel} = change) do
IO.puts(
"Deleting whole funnel '#{change.funnel_name}', there is no way to make it functional without duplicates"
)
Plausible.Repo.delete_all(
from(f in Plausible.Funnel,
where: f.site_id == ^change.site_id,
where: f.id == ^funnel_id
)
)
end
def apply_change(funnel_id, %{update_type: :delete_step} = change) do
IO.puts(
"Deleting step ID:#{change.funnel_step_id} '#{change.goal_name}' - otherwise there will be duplicates in the funnel."
)
step =
Plausible.Repo.get_by!(Plausible.Funnel.Step,
funnel_id: funnel_id,
id: change.funnel_step_id
)
{:ok, _} = Plausible.Repo.delete(step)
end
def apply_change(
_funnel_id,
%{update_type: :update_step, og_goal_id: goal_id, to: goal_id} = change
) do
IO.puts(
"Doing nothing for step '#{change.goal_name}' - the duplicate goal exists, but the update is no-op"
)
:ok
end
def apply_change(funnel_id, %{update_type: :update_step} = change) do
IO.puts(
"Updating step '#{change.goal_name}' from goal_id:#{change.og_goal_id} to goal_id:#{change.to}"
)
step =
Plausible.Repo.get_by!(Plausible.Funnel.Step,
funnel_id: funnel_id,
id: change.funnel_step_id
)
change = Ecto.Changeset.change(step, goal_id: change.to)
{:ok, _} = Plausible.Repo.update(change)
end
def translate_to_db_ops(data) do
for {site, meta} <- data do
{site, to_db_ops(meta)}
end
end
def to_db_ops(meta) do
new_changes =
for {funnel_id, changes} <- meta.changes do
safe_step_updates = changes |> Enum.uniq_by(& &1.to)
steps_to_delete = changes -- safe_step_updates
safe_step_updates =
safe_step_updates
|> Enum.map(&Map.put(&1, :update_type, :update_step))
changes =
Enum.map(steps_to_delete, &Map.put(&1, :update_type, :delete_step)) ++ safe_step_updates
og_steps_count = Enum.at(changes, 0).og_steps_count
steps_after_changes = og_steps_count - Enum.count(steps_to_delete)
if steps_after_changes >= 2 do
{funnel_id, changes}
else
{funnel_id,
[
Enum.at(changes, 0)
|> Map.put(:update_type, :delete_whole_funnel)
]}
end
end
Map.put(meta, :changes, new_changes)
end
def to_map(row) do
[
domain,
site_id,
funnel_name,
funnel_id,
funnel_step_id,
goal_id,
goal_name,
dupe_goal_ids,
all_goal_ids
] = row
%{
domain: domain,
site_id: site_id,
funnel_name: funnel_name,
funnel_id: funnel_id,
funnel_step_id: funnel_step_id,
goal_id: goal_id,
goal_name: goal_name,
dupe_goal_ids: dupe_goal_ids,
all_goal_ids: all_goal_ids
}
end
end

@ -45,21 +45,27 @@ defmodule Plausible.Funnels do
]
def list(%Plausible.Site{id: site_id}) do
Repo.all(
from f in Funnel,
from(f in Funnel,
inner_join: steps in assoc(f, :steps),
where: f.site_id == ^site_id,
select: %{name: f.name, id: f.id, steps_count: count(steps)},
group_by: f.id,
order_by: [desc: :id]
)
)
end
@spec delete(Plausible.Site.t(), pos_integer()) :: :ok
@spec delete(Plausible.Site.t() | pos_integer(), pos_integer()) :: :ok
def delete(%Plausible.Site{id: site_id}, funnel_id) do
delete(site_id, funnel_id)
end
def delete(site_id, funnel_id) do
Repo.delete_all(
from f in Funnel,
from(f in Funnel,
where: f.site_id == ^site_id,
where: f.id == ^funnel_id
)
)
:ok
@ -73,7 +79,7 @@ defmodule Plausible.Funnels do
def get(site_id, funnel_id) when is_integer(site_id) and is_integer(funnel_id) do
q =
from f in Funnel,
from(f in Funnel,
where: f.site_id == ^site_id,
where: f.id == ^funnel_id,
inner_join: steps in assoc(f, :steps),
@ -82,6 +88,7 @@ defmodule Plausible.Funnels do
preload: [
steps: {steps, goal: goal}
]
)
Repo.one(q)
end

@ -0,0 +1,62 @@
WITH DuplicateGoals AS (
SELECT
g.site_id,
CASE
WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
END AS goal_name,
ARRAY_AGG(g.id) AS duplicate_ids
FROM
goals g
WHERE
g.site_id IS NOT NULL
GROUP BY
g.site_id,
goal_name
HAVING COUNT(*) > 1
),
FunnelStepsWithDuplicateGoals AS (
SELECT
fs.id,
fs.goal_id,
fs.funnel_id,
f.name,
CASE
WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
END AS goal_name,
dg.duplicate_ids
FROM
funnel_steps fs
JOIN goals g ON fs.goal_id = g.id
JOIN funnels f ON fs.funnel_id = f.id
JOIN DuplicateGoals dg ON f.site_id = dg.site_id
WHERE
fs.goal_id = ANY(dg.duplicate_ids) -- Ch
)
SELECT
s.domain,
f.site_id,
f.name as funnel_name,
f.id,
fs.id,
fs.goal_id,
fs.goal_name,
fs.duplicate_ids,
array_agg(fs2.goal_id)
FROM
funnels f
JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id
JOIN funnel_steps fs2 ON f.id = fs2.funnel_id
JOIN sites s ON s.id = f.site_id
GROUP BY
s.domain,
f.site_id,
f.name,
f.id,
fs.id,
fs.goal_id,
fs.goal_name,
fs.duplicate_ids
ORDER BY fs.id ASC
;

@ -0,0 +1,124 @@
defmodule Plausible.DataMigration.RewriteFunnelDupesTest do
use Plausible.DataCase, async: true
alias Plausible.DataMigration.RewriteFunnelDupes
import ExUnit.CaptureIO
for goal_type <- ["event_name", "page_path"] do
test "deletes a funnel that cannot be cleaned up from dupe goals (#{goal_type})" do
site = insert(:site)
goals =
setup_goals!(site, [
%{unquote(goal_type) => "/AAA"},
%{unquote(goal_type) => "/AAA"},
%{unquote(goal_type) => "/AAA"}
])
funnel = setup_funnel!(site, "test", goals)
io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
assert io =~ "Processing site ID: #{site.id}"
assert io =~ "Deleting whole funnel"
refute Repo.reload(funnel)
end
test "reduces a funnel if possible (#{goal_type})" do
site = insert(:site)
goals =
setup_goals!(site, [
%{unquote(goal_type) => "/AAA"},
%{unquote(goal_type) => "/BBB"},
%{unquote(goal_type) => "/AAA"},
%{unquote(goal_type) => "/CCC"}
])
funnel = setup_funnel!(site, "test", goals)
io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
assert io =~ "Processing site ID: #{site.id}"
assert io =~ "Deleting step"
assert_funnel(site.id, funnel.id, [
%{unquote(goal_type) => "/AAA"},
%{unquote(goal_type) => "/BBB"},
%{unquote(goal_type) => "/CCC"}
])
end
end
test "dupe names in mixed goal types don't matter" do
site = insert(:site)
gs = [
%{"event_name" => "/AAA"},
%{"page_path" => "/AAA"},
%{"event_name" => "/foo"},
%{"page_path" => "/foo"}
]
goals = setup_goals!(site, gs)
funnel = setup_funnel!(site, "test", goals)
capture_io(fn -> RewriteFunnelDupes.run() end)
assert_funnel(site.id, funnel.id, gs)
end
test "goals across multiple funnels" do
site = insert(:site)
goals = [
%{"event_name" => "/AAA"},
%{"event_name" => "/AAA"},
%{"page_path" => "/foo"},
%{"event_name" => "/AAA"}
]
[g1, g2, g3, g4] = setup_goals!(site, goals)
funnel1 = setup_funnel!(site, "test1", [g1, g2, g3, g4])
funnel2 = setup_funnel!(site, "test2", [g2, g1, g3, g4])
funnel3 = setup_funnel!(site, "test3", [g2, g3])
funnel4 = setup_funnel!(site, "test4", [g2, g1])
capture_io(fn -> RewriteFunnelDupes.run() end)
assert_funnel(site.id, funnel1.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
assert_funnel(site.id, funnel2.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
assert_funnel(site.id, funnel3.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
refute Plausible.Funnels.get(site.id, funnel4.id)
end
defp setup_goals!(site, goals) do
for goal <- goals do
{:ok, g} = Plausible.Goals.create(site, goal)
g
end
end
def setup_funnel!(site, name, goals) do
{:ok, funnel} = Plausible.Funnels.create(site, name, Enum.map(goals, &%{"goal_id" => &1.id}))
funnel
end
def assert_funnel(site_id, funnel_id, goals) do
funnel_goals =
Plausible.Funnels.get(site_id, funnel_id)
|> Map.fetch!(:steps)
|> Enum.map(& &1.goal)
assert Enum.count(funnel_goals) == Enum.count(goals)
Enum.zip(funnel_goals, goals)
|> Enum.each(fn {funnel_goal, expect} ->
key = Map.keys(expect) |> List.first() |> String.to_existing_atom()
assert Map.fetch!(funnel_goal, key) == Map.values(expect) |> List.first()
end)
end
end
Loading…
Cancel
Save