You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_revision_durabilit...

675 lines
23 KiB
Python

"""Regression contracts for durable VPC revision admission and migration."""
from __future__ import annotations
import asyncio
import json
from contextlib import asynccontextmanager
from types import SimpleNamespace
import pytest
from virsh_panel.fabric import FabricNode, FabricPlan
from virsh_panel.vpc import (
VPC_DELETE_RECEIPT_KIND,
VpcCoordinator,
VpcError,
parse_vpc_delete_plan,
parse_vpc_plan,
)
from virsh_panel.vpc_dataplane import derive_private_mac
from test_vpc import (
FakeFabric,
INSTANCE_A,
INSTANCE_B,
MemoryStore,
confirm_delete_hosts,
delete_payload,
payload,
)
GATE = {"active": True, "available": True}
class SimulatedProcessInterruption(BaseException):
"""Model a process exit after a host write has crossed its boundary."""
def _two_host_payload(*, desired_revision=1, operation_id="vpc-op-00000001"):
return payload(
desired_revision=desired_revision,
operation_id=operation_id,
hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
],
)
def _exact_host(plan, host_id, *, pending=None):
member = next(item for item in plan.members if item.host_id == host_id)
return {
"host_id": host_id,
"vpc_id": plan.vpc_id,
"member_ids": [member.member_id],
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"host_config_reusable": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"retiring_members": [],
**({"pending_member_mutations": list(pending)}
if pending is not None else {}),
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": derive_private_mac(plan.vpc_id, member.member_id),
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": False,
"guest_network_verified": False,
"guest_network_readback": "stopped",
"guest_reason_code": "vpc_private_guest_stopped",
"restart_required": False,
}],
"routes": [],
"wg_allowed_ips": [],
"readback": {},
}
def _decode(store, kind, key):
raw = store.get(kind, key)
return json.loads(raw) if raw is not None else None
def test_new_revision_is_durable_with_children_before_first_host_write():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
previous_plan = parse_vpc_plan(payload())
previous = coordinator._desired_record(
previous_plan, GATE, state="applying")
coordinator._save("vpc_desired", previous_plan.vpc_id, previous)
coordinator._save_children(previous)
plan = parse_vpc_plan(_two_host_payload(
desired_revision=2, operation_id="vpc-op-00000002"))
durable_at_write = {}
host_writes = []
async def first_host_write_then_interrupt(*_args, **_kwargs):
host_writes.append("started")
durable_at_write.update({
"root": _decode(store, "vpc_desired", plan.vpc_id),
"member": _decode(
store, "vpc_member", f"{plan.vpc_id}.member-b"),
"host": _decode(store, "vpc_host", f"{plan.vpc_id}.host-b"),
})
raise SimulatedProcessInterruption
coordinator._reconcile_dataplane = first_host_write_then_interrupt
with pytest.raises(SimulatedProcessInterruption):
asyncio.run(coordinator._apply_locked(plan))
assert host_writes == ["started"]
assert durable_at_write["root"]["desired_revision"] == 2
assert durable_at_write["root"]["desired_digest"] == plan.desired_digest
assert durable_at_write["member"]["desired_revision"] == 2
assert durable_at_write["member"]["instance_uuid"] == INSTANCE_B
assert durable_at_write["host"]["desired_revision"] == 2
assert durable_at_write["host"]["pending_member_mutations"] == [INSTANCE_B]
# A process exit cannot run an in-memory finalizer. The pre-write intent
# must therefore remain sufficient for a fresh process to resume revision 2.
assert _decode(store, "vpc_desired", plan.vpc_id)["desired_revision"] == 2
assert _decode(
store, "vpc_member", f"{plan.vpc_id}.member-b")[
"desired_digest"] == plan.desired_digest
def test_retry_preflights_and_locks_full_durable_mutation_closure():
plan = parse_vpc_plan(_two_host_payload())
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
(
FabricNode(
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode(
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
),
"f" * 64,
)
class RetryFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, _plan):
return {
host_id: (
SimpleNamespace(manager=object()),
{"host_id": host_id},
)
for host_id in ("host-a", "host-b")
}
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
MemoryStore(), RetryFabric(hosts=("host-a", "host-b")),
registry=object(), vm_lock=vm_lock,
)
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
record = coordinator._desired_record(plan, GATE, state="applying")
by_host = {host["host_id"]: host for host in record["hosts"]}
# Host A has exact persistent configuration, so only host B belongs to the
# next write selection. Its durable marker remains part of this revision's
# stop-before-write closure until every selected host has converged.
by_host["host-a"].update(
_exact_host(plan, "host-a", pending=[INSTANCE_A]))
checked = []
host_writes = []
def require_stopped(_profile, host_id, instance_uuids):
checked.append((host_id, tuple(instance_uuids)))
if host_id == "host-a":
raise VpcError(
"member became running", 409,
"vpc_member_requires_stopped")
coordinator._require_host_member_mutations_stopped = require_stopped
coordinator._run_host_dataplane = (
lambda *_args, **_kwargs:
host_writes.append(_args[1]) or {})
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, GATE, plan,
host_selection=({"host-b"}, set()),
))
assert acquired == [
("host-a", INSTANCE_A),
("host-b", INSTANCE_B),
]
assert checked == [
("host-a", (INSTANCE_A,)),
("host-b", (INSTANCE_B,)),
]
assert host_writes == []
result_by_host = {host["host_id"]: host for host in record["hosts"]}
assert result_by_host["host-b"]["state"] == "applying"
assert result_by_host["host-b"]["last_error_code"] \
== "vpc_member_requires_stopped"
assert result_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
def test_root_delete_without_prior_record_preflights_full_signed_vm_closure():
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
(
FabricNode(
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode(
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
),
"f" * 64,
)
class DeleteFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, _plan):
return {
host_id: (
SimpleNamespace(manager=object()),
{"host_id": host_id},
)
for host_id in ("host-a", "host-b")
}
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
store = MemoryStore()
coordinator = VpcCoordinator(
store, DeleteFabric(hosts=("host-a", "host-b")),
registry=object(), vm_lock=vm_lock,
)
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
checked = []
host_writes = []
def require_stopped(_profile, host_id, instance_uuids):
checked.append((host_id, tuple(instance_uuids)))
if host_id == "host-b":
raise VpcError(
"member became running", 409,
"vpc_member_requires_stopped")
coordinator._require_host_member_mutations_stopped = require_stopped
coordinator._run_host_dataplane = (
lambda *_args, **_kwargs:
host_writes.append(_args[1]) or {})
target = delete_payload(
vpc_id="vpc-never-applied",
desired_revision=3,
operation_id="vpc-delete-full-stop-fence-0001",
hosts=("host-a", "host-b"),
cidr="10.21.0.0/24",
retiring_members=(
{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": "host-a",
"private_ip_cidr": "10.21.0.10/32",
"private_mac": derive_private_mac(
"vpc-never-applied", "member-a"),
},
{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"host_id": "host-b",
"private_ip_cidr": "10.21.0.11/32",
"private_mac": derive_private_mac(
"vpc-never-applied", "member-b"),
},
),
)
with pytest.raises(VpcError) as blocked:
asyncio.run(coordinator.delete("vpc-never-applied", target))
assert blocked.value.error_code == "vpc_member_requires_stopped"
assert acquired == [
("host-a", INSTANCE_A),
("host-b", INSTANCE_B),
]
assert checked == [
("host-a", (INSTANCE_A,)),
("host-b", (INSTANCE_B,)),
]
assert host_writes == []
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None
durable = _decode(store, "vpc_desired", "vpc-never-applied")
pending = {
host["host_id"]: host["pending_member_mutations"]
for host in durable["hosts"]
}
assert pending == {
"host-a": [INSTANCE_A],
"host-b": [INSTANCE_B],
}
def test_root_delete_partial_retry_ignores_exact_offline_host_fence():
vpc_id = "vpc-partial-delete"
target = delete_payload(
vpc_id=vpc_id,
desired_revision=3,
operation_id="vpc-delete-partial-fence-0001",
hosts=("host-a", "host-b"),
cidr="10.21.0.0/24",
retiring_members=(
{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": "host-a",
"private_ip_cidr": "10.21.0.10/32",
"private_mac": derive_private_mac(vpc_id, "member-a"),
},
{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"host_id": "host-b",
"private_ip_cidr": "10.21.0.11/32",
"private_mac": derive_private_mac(vpc_id, "member-b"),
},
),
)
plan, _manifest = parse_vpc_delete_plan(target)
store = MemoryStore()
first = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")), registry=object())
first._validate_dataplane_resources = lambda _plan: None
async def leave_host_b_pending(
delete_plan, record, _gate, previous_plan=None, **_kwargs):
host_a = next(
host for host in record["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(delete_plan, {"hosts": [host_a]})
host_b = next(
host for host in record["hosts"]
if host["host_id"] == "host-b")
host_b.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_host_unavailable",
})
first._reconcile_dataplane = leave_host_b_pending
with pytest.raises(VpcError) as incomplete:
asyncio.run(first.delete(vpc_id, target))
assert incomplete.value.error_code == "vpc_delete_incomplete"
durable = _decode(store, "vpc_desired", vpc_id)
durable_hosts = {
host["host_id"]: host for host in durable["hosts"]
}
assert first._confirmed_removal(
durable_hosts["host-a"], plan.desired_revision,
plan.desired_digest)
assert {
host_id: durable_hosts[host_id]["pending_member_mutations"]
for host_id in ("host-a", "host-b")
} == {
"host-a": [INSTANCE_A],
"host-b": [INSTANCE_B],
}
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
(
FabricNode(
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode(
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
),
"f" * 64,
)
component_requests = []
class RetryFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, selected):
requested = [node.host_id for node in selected.nodes]
component_requests.append(requested)
if "host-a" in requested:
raise AssertionError(
"exactly removed offline host must not be requested")
return {
"host-b": (
SimpleNamespace(manager=object()),
{"host_id": "host-b"},
),
}
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
retry = VpcCoordinator(
store, RetryFabric(hosts=("host-a", "host-b")),
registry=object(), vm_lock=vm_lock,
)
retry._validate_dataplane_resources = lambda _plan: None
retry._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
retry._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
retry._fabric_peers_for_plan = lambda _plan, _record: []
stopped = []
host_runs = []
def require_stopped(_profile, host_id, instance_uuids):
stopped.append((host_id, tuple(instance_uuids)))
def confirm_host(_plan, host_id, *_args, **_kwargs):
host_runs.append(host_id)
assert host_id == "host-b"
return {
"state": "active",
"removed": True,
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"proxy_ips": [],
"bridge": None,
"bridge_owner": None,
"nft_table": None,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"readback": "confirmed",
"rollback": "none",
"reason_code": None,
}
retry._require_host_member_mutations_stopped = require_stopped
retry._run_host_dataplane = confirm_host
result = asyncio.run(retry.delete(vpc_id, target))
assert component_requests == [["host-b"]]
assert acquired == [("host-b", INSTANCE_B)]
assert stopped == [("host-b", (INSTANCE_B,))]
assert host_runs == ["host-b"]
assert result["deleted"] is True
assert result["removed_host_ids"] == ["host-a", "host-b"]
assert result["readback"] == "confirmed"
receipt = _decode(store, VPC_DELETE_RECEIPT_KIND, vpc_id)
assert receipt["operation_id"] == plan.operation_id
assert receipt["desired_digest"] == plan.desired_digest
def test_legacy_exact_host_migrates_empty_pending_without_host_write():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
plan = parse_vpc_plan(payload())
record = coordinator._desired_record(plan, GATE, state="applying")
record["hosts"] = [_exact_host(plan, "host-a")]
coordinator._finish(record, GATE)
coordinator._save("vpc_desired", plan.vpc_id, record)
coordinator._save_children(record)
host_writes = []
async def unexpected_host_write(*_args, **_kwargs):
host_writes.append(True)
coordinator._reconcile_dataplane = unexpected_host_write
asyncio.run(coordinator.reconcile(plan.vpc_id))
durable = _decode(store, "vpc_desired", plan.vpc_id)
durable_host = _decode(
store, "vpc_host", f"{plan.vpc_id}.host-a")
assert host_writes == []
assert durable["hosts"][0]["pending_member_mutations"] == []
assert durable_host["pending_member_mutations"] == []
def test_explicit_empty_pending_fence_is_not_rederived_from_current_members():
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a",)))
plan = parse_vpc_plan(payload())
record = coordinator._desired_record(plan, GATE, state="applying")
host = record["hosts"][0]
host["pending_member_mutations"] = []
host["config_verified"] = False
host["config_observed_revision"] = 0
host["config_observed_digest"] = None
assert coordinator._migrate_legacy_pending_member_mutations(
record, plan) is False
assert host["pending_member_mutations"] == []
def test_legacy_incomplete_current_and_retiring_hosts_derive_stop_closure():
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
plan = parse_vpc_plan(_two_host_payload())
current = _exact_host(plan, "host-a")
current.update({
"state": "applying",
"config_verified": False,
"host_config_reusable": False,
"config_observed_revision": 0,
"config_observed_digest": None,
"member_nics": [],
})
current_record = coordinator._desired_record(
plan, GATE, state="applying")
current_record["hosts"] = [
current if host["host_id"] == "host-a" else host
for host in current_record["hosts"]
]
assert coordinator._migrate_legacy_pending_member_mutations(
current_record, plan) is True
current_by_host = {
host["host_id"]: host for host in current_record["hosts"]}
assert current_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
retiring = {
"host_id": "host-a",
"vpc_id": plan.vpc_id,
"member_ids": ["member-a"],
"retiring": True,
"retiring_members": [{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": "host-a",
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac(plan.vpc_id, "member-a"),
}],
}
host_b_only = parse_vpc_plan(payload(
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]}],
))
retiring_record = coordinator._desired_record(
host_b_only, GATE, state="applying")
retiring_record["hosts"].append(retiring)
assert coordinator._migrate_legacy_pending_member_mutations(
retiring_record, host_b_only) is True
retiring_by_host = {
host["host_id"]: host for host in retiring_record["hosts"]}
assert retiring_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
def test_legacy_retiring_host_with_malformed_identity_fails_closed():
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
plan = parse_vpc_plan(payload(
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]}],
))
malformed = {
"host_id": "host-a",
"vpc_id": plan.vpc_id,
"member_ids": ["member-a"],
"retiring": True,
"retiring_members": [{
"member_id": "member-a",
"instance_uuid": "not-a-domain-uuid",
"host_id": "host-a",
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac(plan.vpc_id, "member-a"),
}],
}
record = coordinator._desired_record(plan, GATE, state="applying")
record["hosts"].append(malformed)
with pytest.raises(VpcError) as error:
coordinator._migrate_legacy_pending_member_mutations(record, plan)
assert error.value.error_code == "vpc_record_corrupt"
@pytest.mark.parametrize("retiring_members", [None, []])
def test_legacy_retiring_host_requires_complete_member_closure(
retiring_members):
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
plan = parse_vpc_plan(payload(
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]}],
))
retiring = {
"host_id": "host-a",
"vpc_id": plan.vpc_id,
"member_ids": ["member-a"],
"retiring": True,
}
if retiring_members is not None:
retiring["retiring_members"] = retiring_members
record = coordinator._desired_record(plan, GATE, state="applying")
record["hosts"].append(retiring)
with pytest.raises(VpcError) as error:
coordinator._migrate_legacy_pending_member_mutations(record, plan)
assert error.value.error_code == "vpc_record_corrupt"