You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc.py

1448 lines
56 KiB
Python

"""Structured VPC control-plane tests (no host data-plane commands)."""
from __future__ import annotations
import asyncio
import json
import threading
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
import pytest
from virsh_panel.fabric import FabricError, FabricNode, FabricPlan
from virsh_panel.vpc import (
VPC_DELETE_RECEIPT_KIND,
VPC_SCHEMA_VERSION,
VpcCoordinator,
VpcError,
_plan_from_record,
parse_vpc_delete_plan,
parse_vpc_plan,
)
from virsh_panel.vpc_dataplane import derive_private_mac
INSTANCE_A = "11111111-1111-4111-8111-111111111111"
INSTANCE_B = "22222222-2222-4222-8222-222222222222"
class MemoryStore:
def __init__(self):
self.values = {}
self.lock = threading.RLock()
def get(self, kind, key):
with self.lock:
return self.values.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.values[kind, key] = value
def delete(self, kind, key):
with self.lock:
self.values.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.values.items()
if row_kind == kind}
class FakeFabric:
def __init__(self, *, state="active", gate=True, revision=7,
hosts=("host-a", "host-b")):
self.state = state
self.gate = gate
self.revision = revision
self.hosts = hosts
def get_status(self, fabric_id):
if fabric_id == "missing":
raise FabricError("missing", 404, "fabric_not_found")
return {
"fabric_id": fabric_id,
"region": "hk",
"revision": self.revision,
"state": self.state,
"next_stage_allowed": self.gate,
"nodes": [{"host_id": host} for host in self.hosts],
}
def payload(**overrides):
value = {
"schema_version": VPC_SCHEMA_VERSION,
"vpc_id": "vpc-hk-1",
"region": "hk",
"fabric_id": "fabric-hk",
"fabric_revision": 7,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-00000001",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
"routes": [],
"acls": [],
}
value.update(overrides)
return value
def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2,
operation_id="vpc-delete-00000001",
hosts=("host-a",), retiring_members=(), **overrides):
value = payload(
vpc_id=vpc_id,
desired_revision=desired_revision,
operation_id=operation_id,
subnets=[], members=[], routes=[], acls=[],
hosts=[{"host_id": host_id, "members": []} for host_id in hosts],
)
value["retiring_members"] = list(retiring_members)
value.update(overrides)
value["desired_digest"] = parse_vpc_delete_plan(value)[0].desired_digest
return value
def confirm_delete_hosts(plan, record):
"""Fill the exact bounded absence proof required for a delete receipt."""
for host in record.get("hosts", []):
host.update({
"state": "active",
"removed": True,
"transport_verified": True,
"private_dataplane_verified": True,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"member_ids": [],
"retiring_members": [],
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"proxy_ips": [],
"bridge": None,
"bridge_owner": None,
"nft_table": None,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"readback": "confirmed",
"rollback": "none",
})
def retiring_member_a(*, host_id="host-a"):
return {
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": host_id,
"private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
}
def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state():
first = parse_vpc_plan(payload())
second = parse_vpc_plan(payload(
hosts=None,
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.10",
"subnet_id": "subnet-a",
}],
))
assert first.desired_digest == second.desired_digest
assert first.members[0].private_ip == "10.20.0.10"
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="192.0.2.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="10.20.0.0/24",
network_cidr="10.21.0.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(subnets=[
{"subnet_id": "a", "cidr": "10.20.0.0/25"},
{"subnet_id": "b", "cidr": "10.20.0.64/26"},
]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "public", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "0.0.0.0/0",
"destination_cidr": "10.20.0.10/32", "priority": 1,
}]))
def test_member_uuid_matches_existing_libvirt_identity_boundary():
# Existing panel/libvirt records only require the canonical UUID shape;
# operator-supplied UUIDs may use non-RFC version/variant nibbles. The
# engine must accept those persisted VM identities in a VPC DTO too.
legacy_uuid = "11111111-1111-6111-0111-111111111111"
parsed = parse_vpc_plan(payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": legacy_uuid,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
assert parsed.members[0].instance_uuid == legacy_uuid
def test_digest_matches_php_wire_shape_for_host_member_grouping():
wire = payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
)
# PHP computes desired_digest before attaching operation_id to the
# signed/replayed payload; operation_id is an envelope field, not part of
# desired document identity.
digest_wire = {key: value for key, value in wire.items()
if key != "operation_id"}
expected = __import__("hashlib").sha256(
json.dumps(digest_wire, sort_keys=True, separators=(",", ":"),
ensure_ascii=True).encode("utf-8")
).hexdigest()
parsed = parse_vpc_plan(wire)
assert parsed.desired_digest == expected
assert parsed.canonical()["hosts"] == [{
"host_id": "host-a", "members": ["member-a"]
}]
def test_plan_validates_routes_acl_ports_and_member_scope():
parsed = parse_vpc_plan(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32",
"next_hop_member_id": "member-a", "metric": 20}],
acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "port_from": 443,
"port_to": 443, "priority": 1}],
))
assert parsed.host_ids == {"host-a", "host-b"}
assert parsed.routes[0].next_hop_member_id == "member-a"
exact = parse_vpc_plan(payload(routes=[{
"route_id": "exact", "destination_cidr": "10.20.0.10/32",
"next_hop_member_id": "member-a",
}]))
assert exact.routes[0].destination_cidr == "10.20.0.10/32"
high_priority = parse_vpc_plan(payload(acls=[{
"acl_id": "high", "direction": "ingress", "action": "allow",
"protocol": "any", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "priority": 1_000_000,
}]))
assert high_priority.acls[0].priority == 1_000_000
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "bad", "destination_cidr": "0.0.0.0/0",
"next_hop_host_id": "host-a",
}]))
with pytest.raises(VpcError) as broad_route:
parse_vpc_plan(payload(routes=[{
"route_id": "broad", "destination_cidr": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
assert broad_route.value.error_code == "vpc_route_not_exact"
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "bad", "direction": "egress", "action": "allow",
"protocol": "icmp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.0/24", "port_from": 1,
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "alias-conflict",
"destination_cidr": "10.20.0.0/25",
"destination": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(hosts=[{
"host_id": "host-a", "members": ["member-a", "member-a"],
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(
hosts=[],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
def test_coordinator_persists_desired_children_and_blocks_without_fabric():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False))
result = asyncio.run(coordinator.apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["observed_revision"] == 0
assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False
assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"}
assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"}
assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"}
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert result["state"] == "applying"
assert result["next_stage_allowed"] is False
assert result["observed"]["fabric_gate"]["active"] is True
assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True
def test_recovery_reopens_only_the_control_plane_as_applying():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
asyncio.run(coordinator.apply(payload()))
coordinator.recover()
recovered = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert recovered["state"] == "applying"
assert recovered["next_stage_allowed"] is False
assert recovered["hosts"][0]["state"] == "applying"
assert recovered["hosts"][0]["transport_verified"] is True
def test_empty_vpc_cannot_open_fabric_gate():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
hosts=[], members=[],
)))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts"
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
assert result["state"] == "degraded"
assert result["fabric_gate"]["error_code"] == "vpc_no_members"
def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
)))
assert result["state"] == "applying"
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
digest = raw["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
assert observed["next_stage_allowed"] is True
assert observed["hosts"][0]["readback"] == {"safe": "ok"}
fabric.revision = 8
blocked = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert blocked["state"] == "degraded"
assert blocked["next_stage_allowed"] is False
def test_transport_only_observation_never_opens_private_dataplane_gate():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload()))
digest = result["desired_digest"]
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
}))
assert pending["state"] == "applying"
assert pending["next_stage_allowed"] is False
assert pending["private_dataplane_verified"] is False
assert pending["hosts"][0]["state"] == "applying"
assert pending["hosts"][0]["last_error_code"] == (
"vpc_private_dataplane_unverified"
)
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
)))
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert observed["state"] == "degraded"
assert observed["next_stage_allowed"] is False
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": False,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert inconsistent.value.error_code == "vpc_observation_invalid"
def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def wrong_region(fabric_id):
status = original(fabric_id)
status["region"] = "us"
return status
fabric.get_status = wrong_region
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_fabric_region_mismatch"
@pytest.mark.parametrize("mutation", [
lambda status: status.update(fabric_id="other-fabric"),
lambda status: status.update(revision=True),
])
def test_fabric_gate_rejects_tampered_identity_or_revision(mutation):
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def tampered(fabric_id):
status = original(fabric_id)
mutation(status)
return status
fabric.get_status = tampered
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["next_stage_allowed"] is False
assert result["state"] == "degraded"
def test_coordinator_revision_conflict_and_tampered_record_fail_closed():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
first = asyncio.run(coordinator.apply(payload()))
replay = asyncio.run(coordinator.apply(payload()))
assert replay["desired_digest"] == first["desired_digest"]
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.apply(payload(
operation_id="vpc-op-00000002",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert error.value.error_code == "vpc_revision_conflict"
with pytest.raises(VpcError) as invalid_revision:
asyncio.run(coordinator.apply(payload(
desired_revision=0, operation_id="vpc-op-00000003",
)))
assert invalid_revision.value.error_code == "vpc_invalid"
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)))
with pytest.raises(VpcError) as stale:
asyncio.run(coordinator.apply(payload(
desired_revision=1, operation_id="vpc-op-00000005",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert stale.value.error_code == "vpc_revision_conflict"
tampered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
tampered["desired_digest"] = "0" * 64
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as corrupt:
asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert corrupt.value.error_code == "vpc_record_corrupt"
tampered["vpc_id"] = "other-vpc"
tampered["desired_digest"] = first["desired_digest"]
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as mismatched:
coordinator.get_status("vpc-hk-1")
assert mismatched.value.error_code == "vpc_record_corrupt"
# Restore a structurally valid root and corrupt only observed evidence;
# a newer target must not overwrite that evidence silently.
restored = json.loads(store.get("vpc_desired", "vpc-hk-1"))
restored["vpc_id"] = "vpc-hk-1"
restored["desired_revision"] = 2
restored["desired_digest"] = parse_vpc_plan(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)).desired_digest
restored["observed_revision"] = 99
store.set("vpc_desired", "vpc-hk-1", json.dumps(restored))
with pytest.raises(VpcError) as observed_corrupt:
asyncio.run(coordinator.apply(payload(
desired_revision=3, operation_id="vpc-op-00000006",
)))
assert observed_corrupt.value.error_code == "vpc_record_corrupt"
def test_recover_invalidates_observed_gate_and_record_round_trips():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
raw["state"] = "active"
raw["next_stage_allowed"] = True
raw["observed_revision"] = 1
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
transport_verified=True,
private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert recovered["state"] == "uncertain"
assert recovered["next_stage_allowed"] is False
assert recovered["observed_revision"] == 1
assert recovered["observed_digest"] == recovered["desired_digest"]
assert recovered["fabric_gate"]["active"] is False
assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending"
assert recovered["hosts"][0]["observed_revision"] == 1
assert _plan_from_record(recovered).vpc_id == "vpc-hk-1"
def test_removed_children_are_tombstoned_until_observed_cleanup():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
changed = payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}],
members=[],
)
asyncio.run(coordinator.apply(changed))
old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a"))
assert old["state"] == "removed"
assert old["removed"] is True
def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
first = asyncio.run(coordinator.apply(payload()))
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
)))
assert second["state"] == "applying"
assert second["next_stage_allowed"] is False
assert second["observed_revision"] == 1
assert second["observed_digest"] == digest
assert second["hosts"][0]["observed_revision"] == 1
def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
requested = (private_link_states or {}).get(INSTANCE_A, "preserve")
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
"xml_verified": True,
"link_state": "down" if requested == "down" else "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="down",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["readback"] == "confirmed"
assert state["state"] == "suspended"
assert state["vpc_id"] == "vpc-hk-1"
assert state["member_id"] == "member-a"
assert state["link_state"] == "down"
assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a")
# Preserve does not implicitly request link-up from the coordinator.
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="preserve",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["link_state"] == "up"
def test_removed_member_reconcile_uses_durable_retiring_tombstone():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True, member_nics=[],
retiring_members=[],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_state": "removed"},
))
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
assert state["link_state"] == "absent"
def test_instance_reconcile_rejects_stale_revision_before_host_apply():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
calls = []
async def fake_reconcile(*args, **kwargs):
calls.append((args, kwargs))
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={
"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_revision": 99,
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
},
))
assert calls == []
assert state["readback"] == "missing"
assert state["reason_code"] == "vpc_private_expectation_mismatch"
def test_instance_reconcile_rejects_host_mac_drift():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
record["hosts"][0].update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": "02:00:00:00:00:01", "xml_verified": True,
"link_state": "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert state["readback"] == "missing"
assert state["state"] == "uncertain"
def test_removed_member_reconcile_locks_retiring_vm_identity():
store = MemoryStore()
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
# One host pass removes every retiring member. Keep a sibling tombstone
# in the durable row to prove the lock set is not limited to the request
# target.
durable["hosts"][0]["retiring_members"].append({
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"host_id": "host-a", "private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
})
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
acquired.clear()
async def fake_unlocked(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True, member_nics=[],
retiring_members=[],
)
coordinator._reconcile_dataplane_unlocked = fake_unlocked
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a", "desired_state": "removed",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert ("host-a", INSTANCE_A) in acquired
assert ("host-a", INSTANCE_B) in acquired
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
key = f"host-scope.{INSTANCE_A}"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "staged",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
# A same-operation queued/failed row has not established a terminal
# rescue state and must remain a barrier.
for status in ("queued", "running", "exiting", "failed"):
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": status,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict"
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "reinstall_committed": True,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "phase": "result_uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
# Rescue job uncertainty is never exempted by the operation id alone;
# only the vm_rescue active/staged row proves its own terminal XML state.
store.delete("reinstall_job", "job-1")
store.set("rescue_job", "job-2", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
def test_root_delete_accepts_revision_jump_and_superseded_unknown_host():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
asyncio.run(coordinator.apply(payload()))
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append({
"host_ids": sorted(host["host_id"] for host in record["hosts"]),
"retiring": all(host.get("retiring") is True
for host in record["hosts"]),
"operation_id": kwargs.get("operation_id"),
})
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
desired_revision=4,
operation_id="vpc-delete-revision-jump-0001",
hosts=("host-a", "host-b"),
retiring_members=(retiring_member_a(),),
)
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result == {
"vpc_id": "vpc-hk-1",
"operation_id": "vpc-delete-revision-jump-0001",
"desired_revision": 4,
"desired_digest": target["desired_digest"],
"state": "deleted",
"deleted": True,
"readback": "confirmed",
"removed_host_ids": ["host-a", "host-b"],
"completed_at": result["completed_at"],
}
assert result["completed_at"] > 0
assert calls == [{
"host_ids": ["host-a", "host-b"],
"retiring": True,
"operation_id": "vpc-delete-revision-jump-0001",
}]
assert store.get("vpc_desired", "vpc-hk-1") is None
assert store.list("vpc_host") == {}
durable_receipt = json.loads(store.get(
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
assert durable_receipt["readback"] == "confirmed"
assert durable_receipt["completed_at"] == result["completed_at"]
assert "retiring_members" not in durable_receipt
# A receipt written by an earlier engine build is compacted on replay;
# the digest retains the manifest commitment without permanently keeping
# member UUID/private IP/MAC identifiers after successful deletion.
legacy_receipt = dict(durable_receipt)
legacy_receipt["retiring_members"] = target["retiring_members"]
store.set(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1", json.dumps(legacy_receipt))
replay = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert replay == result
compacted_raw = store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")
compacted = json.loads(compacted_raw)
assert "retiring_members" not in compacted
assert INSTANCE_A not in compacted_raw
assert "10.20.0.10" not in compacted_raw
assert retiring_member_a()["private_mac"] not in compacted_raw
conflict = dict(target, operation_id="vpc-delete-revision-jump-0002")
with pytest.raises(VpcError) as changed:
asyncio.run(coordinator.delete("vpc-hk-1", conflict))
assert changed.value.status == 409
assert changed.value.error_code == "vpc_delete_receipt_conflict"
with pytest.raises(VpcError) as recreated:
asyncio.run(coordinator.apply(payload(
desired_revision=5, operation_id="vpc-after-delete-0001")))
assert recreated.value.error_code == "vpc_deleted"
def test_delete_manifest_accepts_unicast_with_odd_last_octet_only():
derived = derive_private_mac("never", "m")
assert derived.endswith(":49")
valid = delete_payload(
vpc_id="never", cidr="10.24.0.0/24", desired_revision=1,
operation_id="vpc-delete-mac-boundary-0001", hosts=("host-a",),
retiring_members=({
"member_id": "m", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.24.0.10/32",
"private_mac": derived,
},),
)
assert parse_vpc_delete_plan(valid)[1][0].private_mac == derived
invalid = dict(valid)
invalid.pop("desired_digest")
invalid["retiring_members"] = [{
**valid["retiring_members"][0],
"private_mac": "13:00:00:00:00:48",
}]
with pytest.raises(VpcError) as multicast:
parse_vpc_delete_plan(invalid)
assert multicast.value.error_code == "vpc_delete_manifest_invalid"
invalid["retiring_members"][0]["private_mac"] = "00:00:00:00:00:00"
with pytest.raises(VpcError) as all_zero:
parse_vpc_delete_plan(invalid)
assert all_zero.value.error_code == "vpc_delete_manifest_invalid"
def test_delete_digest_matches_shared_php_fixture():
fixture = json.loads((
Path(__file__).with_name("fixtures") / "vpc_delete_digest.json"
).read_text(encoding="utf-8"))
plan, retiring = parse_vpc_delete_plan(fixture["document"])
assert plan.desired_digest == fixture["expected_digest"]
assert plan.desired_digest == (
"2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314"
)
assert [host.host_id for host in plan.hosts] == ["11", "2"]
assert [member.member_id for member in retiring] == ["1" * 32]
def test_root_delete_recovers_required_host_after_restart():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-recover-0001",
desired_revision=2,
retiring_members=(retiring_member_a(),),
)
async def interrupted(plan, record, gate, previous_plan=None, **kwargs):
# Simulate a process loss after retiring-member cleanup consumed its
# tombstone but before a host absence proof became durable.
for host in record["hosts"]:
host["retiring_members"] = []
coordinator._reconcile_dataplane = interrupted
with pytest.raises(VpcError) as pending:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert pending.value.status == 503
assert pending.value.error_code == "vpc_delete_incomplete"
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["delete_target"]["required_host_ids"] == ["host-a"]
assert durable["hosts"][0]["retiring_members"] == []
coordinator.recover()
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append({
"retiring": record["hosts"][0].get("retiring"),
"retiring_members": list(
record["hosts"][0].get("retiring_members", [])),
})
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert result["deleted"] is True
assert result["readback"] == "confirmed"
assert calls == [{
"retiring": True,
"retiring_members": [target["retiring_members"][0]],
}]
def test_root_delete_restart_retries_only_hosts_without_exact_removal_proof():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-partial-restart-0001",
desired_revision=2,
hosts=("host-a", "host-b"),
retiring_members=(retiring_member_a(),),
)
calls = []
async def partial_then_confirm(
plan, record, gate, previous_plan=None, **kwargs):
host_ids = [host["host_id"] for host in record["hosts"]]
calls.append(host_ids)
if len(calls) == 1:
assert host_ids == ["host-a", "host-b"]
host_a = next(
host for host in record["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(plan, {"hosts": [host_a]})
host_b = next(
host for host in record["hosts"]
if host["host_id"] == "host-b")
host_b.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_host_unavailable",
})
return
assert host_ids == ["host-b"]
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = partial_then_confirm
with pytest.raises(VpcError) as incomplete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert incomplete.value.error_code == "vpc_delete_incomplete"
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
durable_hosts = {host["host_id"]: host for host in durable["hosts"]}
assert coordinator._confirmed_removal(
durable_hosts["host-a"], 2, target["desired_digest"])
assert not coordinator._confirmed_removal(
durable_hosts["host-b"], 2, target["desired_digest"])
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]}
assert coordinator._confirmed_removal(
recovered_hosts["host-a"], 2, target["desired_digest"])
assert recovered_hosts["host-b"]["state"] == "uncertain"
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert calls == [["host-a", "host-b"], ["host-b"]]
assert result["deleted"] is True
assert result["removed_host_ids"] == ["host-a", "host-b"]
assert result["readback"] == "confirmed"
def test_delete_pending_projection_does_not_probe_confirmed_offline_host():
target = delete_payload(
operation_id="vpc-delete-pending-probe-0001",
desired_revision=2,
hosts=("host-a", "host-b"),
)
plan, _manifest = parse_vpc_delete_plan(target)
nodes = (
FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
)
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820, nodes,
"f" * 64,
)
probed = []
class RetryFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, retry_plan):
for node in retry_plan.nodes:
probed.append(node.host_id)
if node.host_id == "host-a":
raise FabricError(
"confirmed host is now offline", 502,
"fabric_host_capability_probe_failed",
)
return {
"host-b": (
SimpleNamespace(manager=object()),
{"host_id": "host-b"},
),
}
coordinator = VpcCoordinator(
MemoryStore(), RetryFabric(hosts=("host-a", "host-b")),
registry=object(),
)
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
coordinator._run_host_dataplane = lambda *args, **kwargs: {
"state": "active", "removed": True,
"transport_verified": True,
"private_dataplane_verified": True,
"member_nics": [], "routes": [], "wg_allowed_ips": [],
"proxy_ips": [], "bridge": None, "bridge_owner": None,
"nft_table": None, "anti_spoof": False, "rp_filter": None,
"proxy_arp": None, "proxy_arp_pvlan": None,
"readback": "confirmed", "rollback": "none",
}
record = {
"vpc_id": plan.vpc_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"hosts": [{
"host_id": "host-b", "member_ids": [], "retiring": True,
"retiring_members": [],
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
}],
}
asyncio.run(coordinator._reconcile_dataplane_unlocked(
plan, record,
{"active": False, "available": True},
previous_plan=plan,
operation_id=plan.operation_id,
))
assert probed == ["host-b"]
assert coordinator._confirmed_removal(
record["hosts"][0], plan.desired_revision, plan.desired_digest)
def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors():
deterministic_codes = (
"vpc_private_remove_requires_stopped",
"vpc_private_remove_unsafe",
"vpc_private_interface_conflict",
"vpc_seed_missing",
"vpc_seed_readback_invalid",
"vpc_private_xml_invalid",
)
for index, error_code in enumerate(deterministic_codes, start=1):
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id=f"vpc-delete-deterministic-{index:04d}",
retiring_members=(retiring_member_a(),),
)
async def blocked(plan, record, gate, previous_plan=None, **kwargs):
record["hosts"][0].update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": error_code,
})
coordinator._reconcile_dataplane = blocked
with pytest.raises(VpcError) as blocked_delete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert blocked_delete.value.status == 409
assert blocked_delete.value.error_code == error_code
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-transient-error-0001",
retiring_members=(retiring_member_a(),),
)
async def transient(plan, record, gate, previous_plan=None, **kwargs):
record["hosts"][0].update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_private_remove_uncertain",
})
coordinator._reconcile_dataplane = transient
with pytest.raises(VpcError) as transient_delete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert transient_delete.value.status == 503
assert transient_delete.value.error_code == "vpc_delete_incomplete"
@pytest.mark.parametrize("include_proven_host", [False, True])
def test_root_delete_carries_forward_exact_removed_host_without_recontact(
include_proven_host):
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
detached = payload(
desired_revision=2,
operation_id="vpc-member-detach-0001",
subnets=[], members=[], routes=[], acls=[], hosts=[],
)
# Ordinary apply models the earlier member/deployment detach. Its host
# tombstone is then completed with the exact revision/digest proof that a
# lost PHP response may leave behind.
asyncio.run(coordinator.apply(detached))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring"] is True
confirm_delete_hosts(parse_vpc_plan(detached), durable)
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
async def must_not_recontact(*args, **kwargs):
raise AssertionError("exact removed host must not be contacted again")
coordinator._reconcile_dataplane = must_not_recontact
target = delete_payload(
desired_revision=3,
operation_id=("vpc-delete-proven-included-0001"
if include_proven_host
else "vpc-delete-proven-omitted-0001"),
hosts=(("host-a",) if include_proven_host else ()),
)
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result["removed_host_ids"] == ["host-a"]
assert result["readback"] == "confirmed"
def test_root_delete_requires_every_unproven_old_host_in_target():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
desired_revision=2,
operation_id="vpc-delete-missing-host-0001",
hosts=(),
)
with pytest.raises(VpcError) as missing:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert missing.value.status == 409
assert missing.value.error_code == "vpc_delete_host_set_mismatch"
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
def test_root_delete_without_desired_record_proves_requested_hosts_or_empty():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append((
[host["host_id"] for host in record["hosts"]],
[host.get("retiring") for host in record["hosts"]],
[row["instance_uuid"] for host in record["hosts"]
for row in host.get("retiring_members", [])],
))
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
vpc_id="vpc-never-applied",
operation_id="vpc-delete-never-applied-0001",
desired_revision=3,
hosts=("host-a",),
cidr="10.21.0.0/24",
retiring_members=({
"member_id": "member-never-applied",
"instance_uuid": INSTANCE_B,
"host_id": "host-a",
"private_ip_cidr": "10.21.0.10/32",
"private_mac": derive_private_mac(
"vpc-never-applied", "member-never-applied"),
},),
)
result = asyncio.run(coordinator.delete("vpc-never-applied", target))
assert result["removed_host_ids"] == ["host-a"]
assert calls == [(["host-a"], [True], [INSTANCE_B])]
empty = delete_payload(
vpc_id="vpc-empty-never-applied",
operation_id="vpc-delete-empty-never-applied-0001",
desired_revision=1,
hosts=(),
cidr="10.22.0.0/24",
)
result = asyncio.run(coordinator.delete(
"vpc-empty-never-applied", empty))
assert result["removed_host_ids"] == []
assert len(calls) == 1
def test_delete_manifest_removes_unknown_member_nic_before_host_resources(
monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
target = delete_payload(
vpc_id="vpc-remote-only",
operation_id="vpc-delete-remote-only-0001",
desired_revision=5,
hosts=("host-a",),
cidr="10.23.0.0/24",
retiring_members=({
"member_id": "member-remote-only",
"instance_uuid": INSTANCE_B,
"host_id": "host-a",
"private_ip_cidr": "10.23.0.10/32",
"private_mac": derive_private_mac(
"vpc-remote-only", "member-remote-only"),
},),
)
plan, manifest = parse_vpc_delete_plan(target)
rows = [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": member.private_mac,
} for member in manifest]
events = []
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
def remove_nic(self, instance_uuid, **kwargs):
events.append(("nic", instance_uuid, kwargs["vpc_id"]))
return {"removed": True, "readback": "confirmed"}
monkeypatch.setattr(VMService, "remove_private_interface", remove_nic)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile: None,
)
def remove_host(self, dataplane):
events.append(("host", dataplane.host_id, dataplane.vpc.vpc_id))
return {
"state": "active", "removed": True,
"bridge": None, "bridge_owner": None, "nft_table": None,
"anti_spoof": False, "rp_filter": None,
"proxy_arp": None, "proxy_arp_pvlan": None,
"proxy_ips": [], "member_nics": [], "routes": [],
"wg_allowed_ips": [], "readback": "confirmed",
}
monkeypatch.setattr(VpcPrivateHostAdapter, "remove", remove_host)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
observed = coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], None, retiring_members=rows, fabric_public_keys={},
)
assert events == [
("nic", INSTANCE_B, "vpc-remote-only"),
("host", "host-a", "vpc-remote-only"),
]
assert observed["removed"] is True
assert observed["readback"] == "confirmed"
def test_root_delete_receipt_survives_purge_failure_and_keeps_timestamp():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
operation_id="vpc-delete-purge-crash-0001",
retiring_members=(retiring_member_a(),),
)
original_delete = store.delete
failed = False
def fail_once(kind, key):
nonlocal failed
if kind == "vpc_desired" and not failed:
failed = True
raise RuntimeError("crash after receipt commit")
return original_delete(kind, key)
store.delete = fail_once
with pytest.raises(RuntimeError, match="after receipt commit"):
asyncio.run(coordinator.delete("vpc-hk-1", target))
durable_receipt = json.loads(store.get(
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
first_completed_at = durable_receipt["completed_at"]
assert first_completed_at > 0
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result["completed_at"] == first_completed_at
assert result["readback"] == "confirmed"
assert store.get("vpc_desired", "vpc-hk-1") is None