"""Structured VPC control-plane tests (no host data-plane commands).""" from __future__ import annotations import asyncio from concurrent.futures import ThreadPoolExecutor import json import threading import time from contextlib import asynccontextmanager from pathlib import Path from types import SimpleNamespace import pytest from virsh_panel.fabric import ( FabricCoordinator, FabricError, FabricNode, FabricPlan, parse_fabric_plan, ) from virsh_panel.vpc import ( VPC_DELETE_RECEIPT_KIND, VPC_GUEST_EXEC_KIND, VPC_SCHEMA_VERSION, VpcCoordinator, VpcError, _plan_from_record, parse_vpc_delete_plan, parse_vpc_plan, ) from virsh_panel.vpc_dataplane import derive_private_mac INSTANCE_A = "11111111-1111-4111-8111-111111111111" INSTANCE_B = "22222222-2222-4222-8222-222222222222" class MemoryStore: def __init__(self): self.values = {} self.lock = threading.RLock() def get(self, kind, key): with self.lock: return self.values.get((kind, key)) def set(self, kind, key, value): with self.lock: self.values[kind, key] = value def delete(self, kind, key): with self.lock: self.values.pop((kind, key), None) def list(self, kind): with self.lock: return {key: value for (row_kind, key), value in self.values.items() if row_kind == kind} class FakeFabric: def __init__(self, *, state="active", gate=True, revision=7, hosts=("host-a", "host-b"), endpoint_ips=None): self.state = state self.gate = gate self.revision = revision self.hosts = hosts self.endpoint_ips = endpoint_ips or tuple( f"203.0.113.{index + 1}" for index, _host in enumerate(hosts)) def get_status(self, fabric_id): if fabric_id == "missing": raise FabricError("missing", 404, "fabric_not_found") return { "fabric_id": fabric_id, "region": "hk", "revision": self.revision, "state": self.state, "next_stage_allowed": self.gate, "nodes": [ {"host_id": host, "endpoint": f"{self.endpoint_ips[index]}:51820"} for index, host in enumerate(self.hosts) ], } def payload(**overrides): value = { "schema_version": VPC_SCHEMA_VERSION, "vpc_id": "vpc-hk-1", "region": "hk", "fabric_id": "fabric-hk", "fabric_revision": 7, "cidr": "10.20.0.0/24", "desired_revision": 1, "operation_id": "vpc-op-00000001", "subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}], "hosts": [{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}], "routes": [], "acls": [], } value.update(overrides) return value def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2, operation_id="vpc-delete-00000001", hosts=("host-a",), retiring_members=(), **overrides): value = payload( vpc_id=vpc_id, desired_revision=desired_revision, operation_id=operation_id, subnets=[], members=[], routes=[], acls=[], hosts=[{"host_id": host_id, "members": []} for host_id in hosts], ) value["retiring_members"] = list(retiring_members) value.update(overrides) value["desired_digest"] = parse_vpc_delete_plan(value)[0].desired_digest return value def confirm_delete_hosts(plan, record): """Fill the exact bounded absence proof required for a delete receipt.""" for host in record.get("hosts", []): host.update({ "state": "active", "removed": True, "transport_verified": True, "private_dataplane_verified": True, "desired_revision": plan.desired_revision, "observed_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "observed_digest": plan.desired_digest, "member_ids": [], "retiring_members": [], "member_nics": [], "routes": [], "wg_allowed_ips": [], "proxy_ips": [], "bridge": None, "bridge_owner": None, "nft_table": None, "anti_spoof": False, "rp_filter": None, "proxy_arp": None, "proxy_arp_pvlan": None, "readback": "confirmed", "rollback": "none", }) def save_fabric_catalog(store, plan, *, plan_digest=None): """Persist the minimal digest-verified Fabric catalog shape.""" store.set("fabric_desired", plan.fabric_id, json.dumps({ "schema_version": 1, "fabric_id": plan.fabric_id, "region": plan.region, "overlay_cidr": plan.overlay_cidr, "revision": plan.revision, "listen_port": plan.listen_port, "plan_digest": plan.digest if plan_digest is None else plan_digest, "nodes": [ { "host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, } for node in plan.nodes ], })) def retiring_member_a(*, host_id="host-a"): return { "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": host_id, "private_ip_cidr": "10.20.0.10/32", "private_mac": derive_private_mac("vpc-hk-1", "member-a"), } def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state(): first = parse_vpc_plan(payload()) second = parse_vpc_plan(payload( hosts=None, members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.10", "subnet_id": "subnet-a", }], )) assert first.desired_digest == second.desired_digest assert first.members[0].private_ip == "10.20.0.10" with pytest.raises(VpcError): parse_vpc_plan(payload(cidr="192.0.2.0/24")) with pytest.raises(VpcError): parse_vpc_plan(payload(cidr="10.20.0.0/24", network_cidr="10.21.0.0/24")) with pytest.raises(VpcError): parse_vpc_plan(payload(subnets=[ {"subnet_id": "a", "cidr": "10.20.0.0/25"}, {"subnet_id": "b", "cidr": "10.20.0.64/26"}, ])) with pytest.raises(VpcError): parse_vpc_plan(payload(acls=[{ "acl_id": "public", "direction": "ingress", "action": "allow", "protocol": "tcp", "source_cidr": "0.0.0.0/0", "destination_cidr": "10.20.0.10/32", "priority": 1, }])) def test_member_uuid_matches_existing_libvirt_identity_boundary(): # Existing panel/libvirt records only require the canonical UUID shape; # operator-supplied UUIDs may use non-RFC version/variant nibbles. The # engine must accept those persisted VM identities in a VPC DTO too. legacy_uuid = "11111111-1111-6111-0111-111111111111" parsed = parse_vpc_plan(payload( hosts=[{"host_id": "host-a", "members": ["member-a"]}], members=[{ "member_id": "member-a", "instance_uuid": legacy_uuid, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], )) assert parsed.members[0].instance_uuid == legacy_uuid def test_digest_matches_php_wire_shape_for_host_member_grouping(): wire = payload( hosts=[{"host_id": "host-a", "members": ["member-a"]}], members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], ) # PHP computes desired_digest before attaching operation_id to the # signed/replayed payload; operation_id is an envelope field, not part of # desired document identity. digest_wire = {key: value for key, value in wire.items() if key != "operation_id"} expected = __import__("hashlib").sha256( json.dumps(digest_wire, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") ).hexdigest() parsed = parse_vpc_plan(wire) assert parsed.desired_digest == expected assert parsed.canonical()["hosts"] == [{ "host_id": "host-a", "members": ["member-a"] }] def test_plan_validates_routes_acl_ports_and_member_scope(): parsed = parse_vpc_plan(payload( hosts=[ {"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}, {"host_id": "host-b", "members": [ {"member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"}, ]}, ], routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32", "next_hop_member_id": "member-a", "metric": 20}], acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow", "protocol": "tcp", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.10/32", "port_from": 443, "port_to": 443, "priority": 1}], )) assert parsed.host_ids == {"host-a", "host-b"} assert parsed.routes[0].next_hop_member_id == "member-a" exact = parse_vpc_plan(payload(routes=[{ "route_id": "exact", "destination_cidr": "10.20.0.10/32", "next_hop_member_id": "member-a", }])) assert exact.routes[0].destination_cidr == "10.20.0.10/32" high_priority = parse_vpc_plan(payload(acls=[{ "acl_id": "high", "direction": "ingress", "action": "allow", "protocol": "any", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.10/32", "priority": 1_000_000, }])) assert high_priority.acls[0].priority == 1_000_000 with pytest.raises(VpcError): parse_vpc_plan(payload(routes=[{ "route_id": "bad", "destination_cidr": "0.0.0.0/0", "next_hop_host_id": "host-a", }])) with pytest.raises(VpcError) as broad_route: parse_vpc_plan(payload(routes=[{ "route_id": "broad", "destination_cidr": "10.20.0.128/25", "next_hop_member_id": "member-a", }])) assert broad_route.value.error_code == "vpc_route_not_exact" with pytest.raises(VpcError): parse_vpc_plan(payload(acls=[{ "acl_id": "bad", "direction": "egress", "action": "allow", "protocol": "icmp", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.0/24", "port_from": 1, }])) with pytest.raises(VpcError): parse_vpc_plan(payload(routes=[{ "route_id": "alias-conflict", "destination_cidr": "10.20.0.0/25", "destination": "10.20.0.128/25", "next_hop_member_id": "member-a", }])) with pytest.raises(VpcError): parse_vpc_plan(payload(hosts=[{ "host_id": "host-a", "members": ["member-a", "member-a"], }])) with pytest.raises(VpcError): parse_vpc_plan(payload( hosts=[], members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], )) def test_coordinator_persists_desired_children_and_blocks_without_fabric(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False)) result = asyncio.run(coordinator.apply(payload())) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["observed_revision"] == 0 assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"} assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"} assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"} def test_guest_exec_ledger_is_immediate_bounded_private_and_restart_safe(): store = MemoryStore() plan = parse_vpc_plan(payload()) member = plan.members[0] coordinator = VpcCoordinator(store, FakeFabric()) state, persist, clear = coordinator._guest_exec_context(plan, member) assert state == {} core = { "version": 1, "phase": "pending", "domain_uuid": member.instance_uuid, "domain_runtime_id": 7, "qemu_runtime_token": "a" * 32, "guest_network_generation": "generation-1", "probe_kind": "link", "pid": 41, } persist(core) key = f"{plan.vpc_id}.{member.instance_uuid}" encoded = store.get(VPC_GUEST_EXEC_KIND, key) assert encoded is not None assert len(encoded.encode("utf-8")) <= 1024 assert json.loads(encoded)["pid"] == 41 restarted = VpcCoordinator(store, FakeFabric()) loaded, _persist_again, clear_again = restarted._guest_exec_context( plan, member) assert loaded == core asyncio.run(coordinator.apply(payload())) public = json.dumps(coordinator.get_status(plan.vpc_id), sort_keys=True) assert '"pid"' not in public assert "qemu_runtime_token" not in public assert "a" * 32 not in public clear_again() assert store.get(VPC_GUEST_EXEC_KIND, key) is None persist(core) coordinator._purge_deleted_vpc(plan.vpc_id) assert store.get(VPC_GUEST_EXEC_KIND, key) is None def test_guest_exec_ledger_rejects_oversize_or_unknown_fields(): store = MemoryStore() plan = parse_vpc_plan(payload()) member = plan.members[0] key = f"{plan.vpc_id}.{member.instance_uuid}" store.set(VPC_GUEST_EXEC_KIND, key, "x" * 1025) with pytest.raises(VpcError, match="账本损坏"): VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member) bad = { "vpc_id": plan.vpc_id, "host_id": member.host_id, "member_id": member.member_id, "instance_uuid": member.instance_uuid, "version": 1, "phase": "launch_intent", "domain_uuid": member.instance_uuid, "domain_runtime_id": 7, "qemu_runtime_token": "a" * 32, "guest_network_generation": "generation-1", "probe_kind": "link", "unexpected": True, } store.set(VPC_GUEST_EXEC_KIND, key, json.dumps(bad)) with pytest.raises(VpcError, match="账本损坏"): VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member) def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral(): store = MemoryStore() result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) assert result["state"] == "applying" assert result["next_stage_allowed"] is False assert result["observed"]["fabric_gate"]["active"] is True assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True def test_recovery_reopens_only_the_control_plane_as_applying(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) asyncio.run(coordinator.apply(payload())) coordinator.recover() recovered = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert recovered["state"] == "applying" assert recovered["next_stage_allowed"] is False assert recovered["hosts"][0]["state"] == "applying" assert recovered["hosts"][0]["transport_verified"] is True def test_empty_vpc_cannot_open_fabric_gate(): store = MemoryStore() result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload( hosts=[], members=[], ))) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts" result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) assert result["state"] == "degraded" assert result["fabric_gate"]["error_code"] == "vpc_no_members" def test_coordinator_requires_exact_fabric_revision_and_all_host_readback(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a", "host-b")) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload( hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}], ))) assert result["state"] == "applying" raw = json.loads(store.get("vpc_desired", "vpc-hk-1")) digest = raw["desired_digest"] observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True, "readback": {"safe": "ok", "oversized": "x" * 513}}, })) assert observed["state"] == "active" assert observed["next_stage_allowed"] is True assert observed["hosts"][0]["readback"] == {"safe": "ok"} fabric.revision = 8 blocked = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert blocked["state"] == "degraded" assert blocked["next_stage_allowed"] is False def test_transport_only_observation_never_opens_private_dataplane_gate(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload())) digest = result["desired_digest"] pending = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True}, })) assert pending["state"] == "applying" assert pending["next_stage_allowed"] is False assert pending["private_dataplane_verified"] is False assert pending["hosts"][0]["state"] == "applying" assert pending["hosts"][0]["last_error_code"] == ( "vpc_private_dataplane_unverified" ) def test_partial_host_observation_is_degraded_and_never_active(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a", "host-b")) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload( hosts=[ {"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}, {"host_id": "host-b", "members": [ {"member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"}, ]}, ], ))) digest = result["desired_digest"] observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) assert observed["state"] == "degraded" assert observed["next_stage_allowed"] is False with pytest.raises(VpcError) as inconsistent: asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": False, "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) assert inconsistent.value.error_code == "vpc_observation_invalid" def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active(): store = MemoryStore() fabric = FakeFabric() original = fabric.get_status def wrong_region(fabric_id): status = original(fabric_id) status["region"] = "us" return status fabric.get_status = wrong_region with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert error.value.error_code == "vpc_fabric_region_mismatch" assert store.get("vpc_desired", "vpc-hk-1") is None def test_fabric_gate_rejects_underlay_endpoint_inside_tenant_cidr(): store = MemoryStore() fabric = FakeFabric( hosts=("host-a",), endpoint_ips=("10.20.0.50",), ) with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert error.value.error_code == "vpc_fabric_endpoint_overlap" assert store.get("vpc_desired", "vpc-hk-1") is None def test_vpc_rejects_endpoint_from_any_other_fabric_catalog_entry(): store = MemoryStore() other = parse_fabric_plan({ "fabric_id": "fabric-sg", "region": "sg", "overlay_cidr": "100.64.2.0/24", "revision": 3, "nodes": [ { "host_id": "host-c", "overlay_ip": "100.64.2.1", "endpoint": "10.20.0.50:51820", }, { "host_id": "host-d", "overlay_ip": "100.64.2.2", "endpoint": "203.0.113.4:51820", }, ], }) store.set("fabric_desired", other.fabric_id, json.dumps({ "fabric_id": other.fabric_id, "region": other.region, "overlay_cidr": other.overlay_cidr, "revision": other.revision, "listen_port": other.listen_port, "plan_digest": other.digest, "nodes": [ { "host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, } for node in other.nodes ], })) with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) assert error.value.error_code == "vpc_fabric_endpoint_overlap" assert store.get("vpc_desired", "vpc-hk-1") is None def test_vpc_fails_closed_on_corrupt_global_fabric_catalog_entry(): store = MemoryStore() store.set("fabric_desired", "fabric-corrupt", "{}") with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) assert error.value.error_code == "vpc_fabric_record_corrupt" assert store.get("vpc_desired", "vpc-hk-1") is None def test_vpc_fails_closed_on_stale_global_fabric_plan_digest(): store = MemoryStore() other = parse_fabric_plan({ "fabric_id": "fabric-sg", "region": "sg", "overlay_cidr": "100.64.2.0/24", "revision": 3, "nodes": [ { "host_id": "host-c", "overlay_ip": "100.64.2.1", "endpoint": "203.0.113.3:51820", }, { "host_id": "host-d", "overlay_ip": "100.64.2.2", "endpoint": "203.0.113.4:51820", }, ], }) store.set("fabric_desired", other.fabric_id, json.dumps({ "fabric_id": other.fabric_id, "region": other.region, "overlay_cidr": other.overlay_cidr, "revision": other.revision, "listen_port": other.listen_port, "plan_digest": "0" * 64, "nodes": [ { "host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, } for node in other.nodes ], })) with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) assert error.value.error_code == "vpc_fabric_record_corrupt" assert store.get("vpc_desired", "vpc-hk-1") is None def test_vpc_apply_and_fabric_endpoint_change_share_catalog_admission_lock(): store = MemoryStore() current = parse_fabric_plan({ "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 7, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], }) store.set("fabric_desired", current.fabric_id, json.dumps({ "schema_version": 1, "fabric_id": current.fabric_id, "region": current.region, "overlay_cidr": current.overlay_cidr, "revision": current.revision, "listen_port": current.listen_port, "plan_digest": current.digest, "state": "active", "next_stage_allowed": True, "public_keys": { "host-a": "unused-for-catalog-race-a", "host-b": "unused-for-catalog-race-b", }, "nodes": [ { "host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820", "listen_port": 51820, "state": "active", "transport_verified": True, }, { "host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820", "listen_port": 51820, "state": "active", "transport_verified": True, }, ], "updated_at": int(time.time()), })) fabric = FabricCoordinator(store, SimpleNamespace(components={})) coordinator = VpcCoordinator( store, fabric, catalog_lock=fabric.catalog_mutation_lock) entered = asyncio.Event() release = asyncio.Event() original_reconcile = coordinator._reconcile_dataplane async def paused_reconcile(*args, **kwargs): entered.set() await release.wait() return await original_reconcile(*args, **kwargs) coordinator._reconcile_dataplane = paused_reconcile candidate = { "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ { "host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "10.20.0.50:51820", }, { "host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820", }, ], } async def run_race(): vpc_task = asyncio.create_task(coordinator.apply(payload())) await entered.wait() fabric_task = asyncio.create_task(fabric.apply(candidate)) await asyncio.sleep(0) assert fabric_task.done() is False release.set() vpc_result = await vpc_task with pytest.raises(FabricError) as error: await fabric_task return vpc_result, error.value result, error = asyncio.run(run_race()) assert result["fabric_gate"]["active"] is True assert error.error_code == "fabric_endpoint_vpc_conflict" saved = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert saved["cidr"] == "10.20.0.0/24" def test_fabric_first_endpoint_change_blocks_conflicting_vpc_admission(): store = MemoryStore() lock_owner = FabricCoordinator(store, SimpleNamespace(components={})) gate_fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator( store, gate_fabric, catalog_lock=lock_owner.catalog_mutation_lock, ) entered = asyncio.Event() release = asyncio.Event() async def publish_endpoint(_plan): entered.set() await release.wait() gate_fabric.endpoint_ips = ("10.20.0.50",) return {"state": "active"} lock_owner._apply_locked = publish_endpoint candidate = { "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ { "host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "10.20.0.50:51820", }, { "host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820", }, ], } async def run_race(): fabric_task = asyncio.create_task(lock_owner.apply(candidate)) await entered.wait() vpc_task = asyncio.create_task(coordinator.apply(payload())) await asyncio.sleep(0) assert vpc_task.done() is False release.set() await fabric_task with pytest.raises(VpcError) as error: await vpc_task return error.value error = asyncio.run(run_race()) assert error.error_code == "vpc_fabric_endpoint_overlap" assert store.get("vpc_desired", "vpc-hk-1") is None def test_cancelled_fabric_apply_drains_thread_before_releasing_catalog_lock(): store = MemoryStore() fabric = FabricCoordinator(store, SimpleNamespace(components={})) coordinator = VpcCoordinator( store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock) entered = threading.Event() release = threading.Event() async def blocked_apply(_plan): def mutate(): entered.set() assert release.wait(timeout=2) store.set("test", "fabric-thread", "settled") await asyncio.to_thread(mutate) return {"state": "settled"} fabric._apply_locked = blocked_apply candidate = { "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ { "host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820", }, { "host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820", }, ], } async def run_cancel(): fabric_task = asyncio.create_task(fabric.apply(candidate)) for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.01) assert entered.is_set() vpc_task = asyncio.create_task(coordinator.apply(payload())) await asyncio.sleep(0) assert vpc_task.done() is False fabric_task.cancel() await asyncio.sleep(0) assert fabric_task.done() is False assert vpc_task.done() is False assert store.get("test", "fabric-thread") is None release.set() with pytest.raises(asyncio.CancelledError): await fabric_task return await asyncio.wait_for(vpc_task, timeout=1) result = asyncio.run(run_cancel()) assert store.get("test", "fabric-thread") == "settled" assert result["vpc_id"] == "vpc-hk-1" assert store.get("vpc_desired", "vpc-hk-1") is not None def test_cancelled_vpc_apply_drains_thread_and_saves_before_fabric_enters(): store = MemoryStore() fabric = FabricCoordinator(store, SimpleNamespace(components={})) coordinator = VpcCoordinator( store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock) vpc_thread_entered = threading.Event() vpc_thread_release = threading.Event() fabric_entered = asyncio.Event() async def blocked_reconcile(_plan, record, _gate, _previous_plan): def mutate(): vpc_thread_entered.set() assert vpc_thread_release.wait(timeout=2) record["thread_mutation_settled"] = True await asyncio.to_thread(mutate) async def observed_fabric_apply(_plan): fabric_entered.set() return {"state": "settled"} coordinator._reconcile_dataplane = blocked_reconcile fabric._apply_locked = observed_fabric_apply candidate = { "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ { "host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820", }, { "host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820", }, ], } async def run_cancel(): vpc_task = asyncio.create_task(coordinator.apply(payload())) for _attempt in range(100): if vpc_thread_entered.is_set(): break await asyncio.sleep(0.01) assert vpc_thread_entered.is_set() fabric_task = asyncio.create_task(fabric.apply(candidate)) await asyncio.sleep(0) assert fabric_entered.is_set() is False vpc_task.cancel() await asyncio.sleep(0) assert vpc_task.done() is False assert fabric_entered.is_set() is False assert store.get("vpc_desired", "vpc-hk-1") is None vpc_thread_release.set() with pytest.raises(asyncio.CancelledError): await vpc_task await asyncio.wait_for(fabric_task, timeout=1) asyncio.run(run_cancel()) saved = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert saved["thread_mutation_settled"] is True def test_cancelled_root_delete_drains_host_mutation_before_catalog_unlock(): store = MemoryStore() fabric = FabricCoordinator(store, SimpleNamespace(components={})) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), catalog_lock=fabric.catalog_mutation_lock, ) delete_thread_entered = threading.Event() delete_thread_release = threading.Event() fabric_entered = asyncio.Event() async def blocked_delete(plan, record, _gate, _previous_plan, **_kwargs): def mutate(): delete_thread_entered.set() assert delete_thread_release.wait(timeout=2) confirm_delete_hosts(plan, record) await asyncio.to_thread(mutate) async def observed_fabric_apply(_plan): fabric_entered.set() return {"state": "settled"} coordinator._reconcile_dataplane = blocked_delete fabric._apply_locked = observed_fabric_apply target = delete_payload( vpc_id="vpc-never-applied", desired_revision=3, operation_id="vpc-delete-cancel-drain-0001", cidr="10.21.0.0/24", ) candidate = { "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], } async def run_cancel(): delete_task = asyncio.create_task(coordinator.delete( "vpc-never-applied", target)) for _attempt in range(100): if delete_thread_entered.is_set(): break await asyncio.sleep(0.01) assert delete_thread_entered.is_set() # The host-bearing absent-target delete has already published its # temporary CIDR reservation, but still owns the shared catalog lock. assert store.get("vpc_desired", "vpc-never-applied") is not None fabric_task = asyncio.create_task(fabric.apply(candidate)) await asyncio.sleep(0) assert fabric_entered.is_set() is False delete_task.cancel() await asyncio.sleep(0) assert delete_task.done() is False assert fabric_entered.is_set() is False assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None delete_thread_release.set() with pytest.raises(asyncio.CancelledError): await delete_task await asyncio.wait_for(fabric_task, timeout=1) asyncio.run(run_cancel()) receipt = json.loads(store.get( VPC_DELETE_RECEIPT_KIND, "vpc-never-applied")) assert receipt["state"] == "deleted" assert receipt["readback"] == "confirmed" assert store.get("vpc_desired", "vpc-never-applied") is None assert fabric_entered.is_set() is True @pytest.mark.parametrize("entrypoint", ["observe", "reconcile", "instance"]) def test_fabric_gate_change_precedes_every_vpc_evidence_publication( entrypoint): store = MemoryStore() lock_owner = FabricCoordinator(store, SimpleNamespace(components={})) gate_fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator( store, gate_fabric, catalog_lock=lock_owner.catalog_mutation_lock, ) initial = asyncio.run(coordinator.apply(payload())) digest = initial["desired_digest"] active_observation = { "host-a": { "state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True, "config_verified": True, "config_observed_revision": 1, "config_observed_digest": digest, }, } active = asyncio.run(coordinator.observe( "vpc-hk-1", active_observation)) assert active["next_stage_allowed"] is True async def run_race(): fabric_entered = asyncio.Event() release_fabric = asyncio.Event() async def change_fabric_gate(): async with lock_owner.catalog_mutation_lock(): fabric_entered.set() await release_fabric.wait() gate_fabric.revision = 8 gate_fabric.state = "degraded" gate_fabric.gate = False fabric_task = asyncio.create_task(change_fabric_gate()) await fabric_entered.wait() if entrypoint == "observe": vpc_task = asyncio.create_task(coordinator.observe( "vpc-hk-1", active_observation)) elif entrypoint == "reconcile": vpc_task = asyncio.create_task(coordinator.reconcile("vpc-hk-1")) else: vpc_task = asyncio.create_task(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={ "required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", }, )) await asyncio.sleep(0) assert vpc_task.done() is False release_fabric.set() await fabric_task return await asyncio.wait_for(vpc_task, timeout=1) result = asyncio.run(run_race()) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["fabric_gate"]["fabric_revision"] == 8 assert durable["fabric_gate"]["active"] is False assert durable["next_stage_allowed"] is False assert durable["state"] != "active" if entrypoint == "instance": assert result["readback"] != "confirmed" else: assert result["next_stage_allowed"] is False def test_observe_locks_active_and_retiring_vm_identities_in_sorted_order(): store = MemoryStore() acquired = [] @asynccontextmanager async def vm_lock(host_id, instance_uuid): acquired.append((host_id, instance_uuid)) yield coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock) asyncio.run(coordinator.apply(payload(hosts=[{ "host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, {"member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"}, ], }]))) second = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-observe-lock-order-0002", ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["hosts"][0]["retiring_members"][0][ "instance_uuid"] == INSTANCE_B acquired.clear() asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": { "state": "active", "observed_revision": 2, "observed_digest": second["desired_digest"], "transport_verified": True, "private_dataplane_verified": True, }, })) assert acquired == [ ("host-a", INSTANCE_A), ("host-a", INSTANCE_B), ] @pytest.mark.parametrize("entrypoint", ["reconcile", "instance"]) def test_cancelled_vpc_retry_drains_host_thread_before_catalog_unlock( entrypoint): store = MemoryStore() lock_owner = FabricCoordinator(store, SimpleNamespace(components={})) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), catalog_lock=lock_owner.catalog_mutation_lock, ) asyncio.run(coordinator.apply(payload())) thread_entered = threading.Event() thread_release = threading.Event() async def blocked_reconcile(_plan, record, _gate, _previous_plan, **_kwargs): def mutate(): thread_entered.set() assert thread_release.wait(timeout=2) record["hosts"][0]["cancel_drain_marker"] = entrypoint await asyncio.to_thread(mutate) coordinator._reconcile_dataplane = blocked_reconcile async def run_cancel(): fabric_entered = asyncio.Event() async def observed_fabric_apply(_plan): fabric_entered.set() return {"state": "settled"} lock_owner._apply_locked = observed_fabric_apply if entrypoint == "reconcile": vpc_task = asyncio.create_task( coordinator.reconcile("vpc-hk-1")) else: vpc_task = asyncio.create_task(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={ "required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", }, )) for _attempt in range(100): if thread_entered.is_set(): break await asyncio.sleep(0.01) assert thread_entered.is_set() fabric_task = asyncio.create_task(lock_owner.apply({ "fabric_id": "fabric-hk", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 8, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], })) await asyncio.sleep(0) assert fabric_entered.is_set() is False vpc_task.cancel() await asyncio.sleep(0) assert vpc_task.done() is False assert fabric_entered.is_set() is False thread_release.set() with pytest.raises(asyncio.CancelledError): await vpc_task await asyncio.wait_for(fabric_task, timeout=1) assert fabric_entered.is_set() is True asyncio.run(run_cancel()) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["hosts"][0]["cancel_drain_marker"] == entrypoint def test_fabric_gate_rejects_tampered_identity_without_saving(): store = MemoryStore() fabric = FakeFabric() original = fabric.get_status def tampered(fabric_id): status = original(fabric_id) status["fabric_id"] = "other-fabric" return status fabric.get_status = tampered with pytest.raises(VpcError) as error: asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert error.value.error_code == "vpc_fabric_identity_mismatch" assert store.get("vpc_desired", "vpc-hk-1") is None def test_fabric_gate_keeps_revision_mismatch_as_reconcilable_state(): store = MemoryStore() fabric = FakeFabric() original = fabric.get_status def tampered(fabric_id): status = original(fabric_id) status["revision"] = True return status fabric.get_status = tampered result = asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert result["next_stage_allowed"] is False assert result["state"] == "degraded" def test_coordinator_revision_conflict_and_tampered_record_fail_closed(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) first = asyncio.run(coordinator.apply(payload())) replay = asyncio.run(coordinator.apply(payload())) assert replay["desired_digest"] == first["desired_digest"] with pytest.raises(VpcError) as error: asyncio.run(coordinator.apply(payload( operation_id="vpc-op-00000002", members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}], hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}, ]}], ))) assert error.value.error_code == "vpc_revision_conflict" with pytest.raises(VpcError) as invalid_revision: asyncio.run(coordinator.apply(payload( desired_revision=0, operation_id="vpc-op-00000003", ))) assert invalid_revision.value.error_code == "vpc_invalid" asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000004", ))) with pytest.raises(VpcError) as stale: asyncio.run(coordinator.apply(payload( desired_revision=1, operation_id="vpc-op-00000005", members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}], hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}, ]}], ))) assert stale.value.error_code == "vpc_revision_conflict" tampered = json.loads(store.get("vpc_desired", "vpc-hk-1")) tampered["desired_digest"] = "0" * 64 store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered)) with pytest.raises(VpcError) as corrupt: asyncio.run(coordinator.reconcile("vpc-hk-1")) assert corrupt.value.error_code == "vpc_record_corrupt" tampered["vpc_id"] = "other-vpc" tampered["desired_digest"] = first["desired_digest"] store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered)) with pytest.raises(VpcError) as mismatched: coordinator.get_status("vpc-hk-1") assert mismatched.value.error_code == "vpc_record_corrupt" # Restore a structurally valid root and corrupt only observed evidence; # a newer target must not overwrite that evidence silently. restored = json.loads(store.get("vpc_desired", "vpc-hk-1")) restored["vpc_id"] = "vpc-hk-1" restored["desired_revision"] = 2 restored["desired_digest"] = parse_vpc_plan(payload( desired_revision=2, operation_id="vpc-op-00000004", )).desired_digest restored["observed_revision"] = 99 store.set("vpc_desired", "vpc-hk-1", json.dumps(restored)) with pytest.raises(VpcError) as observed_corrupt: asyncio.run(coordinator.apply(payload( desired_revision=3, operation_id="vpc-op-00000006", ))) assert observed_corrupt.value.error_code == "vpc_record_corrupt" def test_recover_invalidates_observed_gate_and_record_round_trips(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) raw = json.loads(store.get("vpc_desired", "vpc-hk-1")) raw["state"] = "active" raw["next_stage_allowed"] = True raw["observed_revision"] = 1 raw["observed_digest"] = raw["desired_digest"] raw["hosts"][0].update(state="active", observed_revision=1, observed_digest=raw["desired_digest"], transport_verified=True, private_dataplane_verified=True) store.set("vpc_desired", "vpc-hk-1", json.dumps(raw)) coordinator.recover() recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert recovered["state"] == "uncertain" assert recovered["next_stage_allowed"] is False assert recovered["observed_revision"] == 1 assert recovered["observed_digest"] == recovered["desired_digest"] assert recovered["fabric_gate"]["active"] is False assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending" assert recovered["hosts"][0]["state"] == "uncertain" assert recovered["hosts"][0]["transport_verified"] is False assert recovered["hosts"][0]["private_dataplane_verified"] is False assert recovered["hosts"][0]["observed_revision"] == 1 assert _plan_from_record(recovered).vpc_id == "vpc-hk-1" def test_removed_children_are_tombstoned_until_observed_cleanup(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) changed = payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ) asyncio.run(coordinator.apply(changed)) old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a")) assert old["state"] == "removed" assert old["removed"] is True def test_new_revision_drops_host_with_exact_prior_removal_proof(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) first_payload = payload( hosts=[{"host_id": "host-a", "members": []}], members=[]) asyncio.run(coordinator.apply(first_payload)) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) confirm_delete_hosts(parse_vpc_plan(first_payload), durable) store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) probed = [] async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs): probed.extend(host["host_id"] for host in record["hosts"]) coordinator._reconcile_dataplane = capture_hosts result = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ))) assert probed == ["host-b"] assert [host["host_id"] for host in result["hosts"]] == ["host-b"] saved = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert [host["host_id"] for host in saved["hosts"]] == ["host-b"] def test_new_revision_retires_host_without_exact_prior_removal_proof(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) first_payload = payload() asyncio.run(coordinator.apply(first_payload)) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) # A marker without the exact typed absence/readback fields is not proof. durable["hosts"][0]["removed"] = True store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) probed = [] async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs): probed.extend(host["host_id"] for host in record["hosts"]) coordinator._reconcile_dataplane = capture_hosts result = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ))) assert probed == ["host-b", "host-a"] second_durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) retiring = next(host for host in second_durable["hosts"] if host["host_id"] == "host-a") assert retiring["retiring"] is True assert retiring.get("removed") is not True assert retiring["member_ids"] == ["member-a"] assert retiring["retiring_members"] == [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", "private_mac": derive_private_mac("vpc-hk-1", "member-a"), }] probed.clear() third_payload = payload( desired_revision=3, operation_id="vpc-op-00000003", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ) third = asyncio.run(coordinator.apply(third_payload)) assert probed == ["host-b", "host-a"] third_durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) carried = next(host for host in third_durable["hosts"] if host["host_id"] == "host-a") assert carried["retiring"] is True assert carried.get("removed") is not True assert carried["member_ids"] == retiring["member_ids"] assert carried["retiring_members"] == retiring["retiring_members"] child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a")) assert child["state"] == "deleting" assert child.get("removed") is not True durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) host_a = next(host for host in durable["hosts"] if host["host_id"] == "host-a") confirm_delete_hosts(parse_vpc_plan(third_payload), {"hosts": [host_a]}) store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) coordinator._save_children(durable) probed.clear() fourth = asyncio.run(coordinator.apply(payload( desired_revision=4, operation_id="vpc-op-00000004", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ))) assert probed == ["host-b"] assert [host["host_id"] for host in fourth["hosts"]] == ["host-b"] child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a")) assert coordinator._terminal_removal(child) assert child["desired_revision"] == 3 assert child["observed_revision"] == 3 root_delete_calls = [] async def confirm_root_delete( plan, record, gate, previous_plan=None, **kwargs): root_delete_calls.append( [host["host_id"] for host in record["hosts"]]) confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = confirm_root_delete member_b = { "member_id": "member-b", "instance_uuid": INSTANCE_B, "host_id": "host-b", "private_ip_cidr": "10.20.0.11/32", "private_mac": derive_private_mac("vpc-hk-1", "member-b"), } deleted = asyncio.run(coordinator.delete("vpc-hk-1", delete_payload( desired_revision=5, operation_id="vpc-delete-historical-proof-0001", hosts=("host-a", "host-b"), retiring_members=(retiring_member_a(), member_b), ))) assert root_delete_calls == [["host-b"]] assert deleted["removed_host_ids"] == ["host-a", "host-b"] @pytest.mark.parametrize("corruption", ["not-list", "non-object", "missing"]) def test_new_revision_rejects_corrupt_durable_retiring_member_closure(corruption): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) asyncio.run(coordinator.apply(payload())) async def leave_pending(plan, record, gate, previous_plan=None, **kwargs): return None coordinator._reconcile_dataplane = leave_pending asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) host_a = next(host for host in durable["hosts"] if host["host_id"] == "host-a") if corruption == "not-list": host_a["retiring_members"] = {"member_id": "member-a"} elif corruption == "non-object": host_a["retiring_members"] = [None] else: host_a["retiring_members"] = [] tampered = json.dumps(durable) store.set("vpc_desired", "vpc-hk-1", tampered) with pytest.raises(VpcError) as rejected: asyncio.run(coordinator.apply(payload( desired_revision=3, operation_id="vpc-op-00000003", hosts=[{"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ))) assert rejected.value.error_code == "vpc_record_corrupt" assert store.get("vpc_desired", "vpc-hk-1") == tampered def test_apply_replay_retries_only_hosts_without_exact_removal_proof_after_restart(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) target = payload(hosts=[ {"host_id": "host-a", "members": []}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) calls = [] async def remove_a_then_recover_b( plan, record, gate, previous_plan=None, **kwargs): host_ids = [host["host_id"] for host in record["hosts"]] calls.append(host_ids) if len(calls) == 1: assert host_ids == ["host-a", "host-b"] host_a = next(host for host in record["hosts"] if host["host_id"] == "host-a") confirm_delete_hosts(plan, {"hosts": [host_a]}) host_b = next(host for host in record["hosts"] if host["host_id"] == "host-b") host_b.update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": "vpc_host_unavailable", }) return # Host A is now offline. Reaching this branch with A in the execution # closure would model the liveness regression this test prevents. assert host_ids == ["host-b"] record["hosts"][0].update({ "state": "active", "removed": False, "transport_verified": True, "private_dataplane_verified": True, "desired_revision": plan.desired_revision, "observed_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "observed_digest": plan.desired_digest, "last_error_code": None, }) coordinator._reconcile_dataplane = remove_a_then_recover_b first = asyncio.run(coordinator.apply(target)) assert first["state"] != "active" durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) durable_hosts = {host["host_id"]: host for host in durable["hosts"]} assert coordinator._confirmed_removal( durable_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest) assert durable_hosts["host-b"]["state"] == "uncertain" coordinator.recover() recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]} assert coordinator._confirmed_removal( recovered_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest) assert recovered_hosts["host-b"]["state"] == "uncertain" result = asyncio.run(coordinator.apply(target)) assert calls == [["host-a", "host-b"], ["host-b"]] assert result["state"] == "active" assert result["next_stage_allowed"] is True final_hosts = {host["host_id"]: host for host in result["hosts"]} assert coordinator._confirmed_removal( final_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest) assert final_hosts["host-b"]["state"] == "active" def test_reconcile_after_restart_retries_only_pending_normal_apply_hosts(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) target = payload(hosts=[ {"host_id": "host-a", "members": []}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) async def partial(plan, record, gate, previous_plan=None, **kwargs): host_a = next(host for host in record["hosts"] if host["host_id"] == "host-a") confirm_delete_hosts(plan, {"hosts": [host_a]}) host_b = next(host for host in record["hosts"] if host["host_id"] == "host-b") host_b.update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": "vpc_host_unavailable", }) coordinator._reconcile_dataplane = partial asyncio.run(coordinator.apply(target)) coordinator.recover() restarted = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) calls = [] async def recover_b(plan, record, gate, previous_plan=None, **kwargs): host_ids = [host["host_id"] for host in record["hosts"]] calls.append(host_ids) # Host A is intentionally unreachable after restart. It must not be # part of the execution closure once its exact absence proof is # durable; only the previously unavailable host B may be contacted. assert host_ids == ["host-b"] record["hosts"][0].update({ "state": "active", "removed": False, "transport_verified": True, "private_dataplane_verified": True, "desired_revision": plan.desired_revision, "observed_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "observed_digest": plan.desired_digest, "last_error_code": None, }) restarted._reconcile_dataplane_unlocked = recover_b result = asyncio.run(restarted.reconcile("vpc-hk-1")) assert calls == [["host-b"]] assert result["state"] == "active" assert result["next_stage_allowed"] is True final_hosts = {host["host_id"]: host for host in result["hosts"]} plan = parse_vpc_plan(target) assert restarted._confirmed_removal( final_hosts["host-a"], plan.desired_revision, plan.desired_digest) assert final_hosts["host-b"]["state"] == "active" def test_new_desired_revision_keeps_last_observed_evidence_until_readback(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) first = asyncio.run(coordinator.apply(payload())) digest = first["desired_digest"] asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True}, })) second = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", ))) assert second["state"] == "applying" assert second["next_stage_allowed"] is False assert second["observed_revision"] == 1 assert second["observed_digest"] == digest assert second["hosts"][0]["observed_revision"] == 1 def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): host = record["hosts"][0] requested = (private_link_states or {}).get(INSTANCE_A, "preserve") host.update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, config_verified=True, config_observed_revision=plan.desired_revision, config_observed_digest=plan.desired_digest, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": derive_private_mac("vpc-hk-1", "member-a"), "xml_verified": True, "config_verified": True, "seed_verified": True, "restart_required": False, "domain_running": True, "guest_network_verified": requested != "down", "guest_network_readback": ( "confirmed" if requested != "down" else "mismatch"), "guest_reason_code": ( None if requested != "down" else "vpc_private_guest_link_mismatch"), "guest_private_mac": (derive_private_mac( "vpc-hk-1", "member-a") if requested != "down" else None), "guest_private_ip_cidr": ( "10.20.0.10/32" if requested != "down" else None), "guest_link_state": ( "up" if requested != "down" else None), "guest_routes": [], "link_state": "down" if requested == "down" else "up", }], ) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="down", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert state["readback"] == "confirmed" assert state["state"] == "suspended" assert state["vpc_id"] == "vpc-hk-1" assert state["member_id"] == "member-a" assert state["link_state"] == "down" assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a") # Preserve does not implicitly request link-up from the coordinator. state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="preserve", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert state["link_state"] == "up" def test_instance_reconcile_invalidates_stale_proof_when_fabric_is_inactive(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def confirm(plan, record, gate, previous_plan=None, private_link_states=None): record["hosts"][0].update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, config_verified=True, config_observed_revision=plan.desired_revision, config_observed_digest=plan.desired_digest, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": derive_private_mac( "vpc-hk-1", "member-a"), "xml_verified": True, "config_verified": True, "seed_verified": True, "restart_required": False, "domain_running": True, "guest_network_verified": True, "guest_network_readback": "confirmed", "guest_reason_code": None, "guest_private_mac": derive_private_mac( "vpc-hk-1", "member-a"), "guest_private_ip_cidr": "10.20.0.10/32", "guest_link_state": "up", "guest_routes": [], "link_state": "up", }], ) coordinator._reconcile_dataplane = confirm active = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert active["readback"] == "confirmed" coordinator.fabric = FakeFabric( state="degraded", gate=False, hosts=("host-a",)) coordinator._reconcile_dataplane = ( VpcCoordinator._reconcile_dataplane.__get__( coordinator, VpcCoordinator)) stale = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert stale["state"] == "uncertain" assert stale["readback"] == "missing" def test_removed_member_reconcile_uses_durable_retiring_tombstone(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="absent", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_state": "removed"}, )) assert state["state"] == "removed" assert state["readback"] == "confirmed" assert state["link_state"] == "absent" def test_instance_reconcile_rejects_stale_revision_before_host_apply(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) calls = [] async def fake_reconcile(*args, **kwargs): calls.append((args, kwargs)) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={ "required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_revision": 99, "private_mac": derive_private_mac("vpc-hk-1", "member-a"), }, )) assert calls == [] assert state["readback"] == "missing" assert state["reason_code"] == "vpc_private_expectation_mismatch" def test_instance_reconcile_rejects_host_mac_drift(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): record["hosts"][0].update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, config_verified=True, config_observed_revision=plan.desired_revision, config_observed_digest=plan.desired_digest, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": "02:00:00:00:00:01", "xml_verified": True, "config_verified": True, "seed_verified": True, "restart_required": False, "domain_running": True, "guest_network_verified": True, "guest_network_readback": "confirmed", "guest_reason_code": None, "guest_private_mac": "02:00:00:00:00:01", "guest_private_ip_cidr": "10.20.0.10/32", "guest_link_state": "up", "guest_routes": [], "link_state": "up", }], ) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "private_mac": derive_private_mac("vpc-hk-1", "member-a")}, )) assert state["readback"] == "missing" assert state["state"] == "uncertain" def test_removed_member_reconcile_locks_retiring_vm_identity(): store = MemoryStore() acquired = [] @asynccontextmanager async def vm_lock(host_id, instance_uuid): acquired.append((host_id, instance_uuid)) yield coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock) asyncio.run(coordinator.apply(payload())) asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) # One host pass removes every retiring member. Keep a sibling tombstone # in the durable row to prove the lock set is not limited to the request # target. durable["hosts"][0]["retiring_members"].append({ "member_id": "member-b", "instance_uuid": INSTANCE_B, "host_id": "host-a", "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", "private_mac": derive_private_mac("vpc-hk-1", "member-b"), }) store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) acquired.clear() async def fake_unlocked(plan, record, gate, previous_plan=None, private_link_states=None): confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane_unlocked = fake_unlocked state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="absent", expected={"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_state": "removed", "private_mac": derive_private_mac("vpc-hk-1", "member-a")}, )) assert ("host-a", INSTANCE_A) in acquired assert ("host-a", INSTANCE_B) in acquired assert state["state"] == "removed" assert state["readback"] == "confirmed" def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation(): store = MemoryStore() registry = SimpleNamespace( components={"host-a": SimpleNamespace(store=store)}, ) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), registry=registry, ) key = f"host-scope.{INSTANCE_A}" store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "active", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict" store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "staged", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None # A same-operation queued/failed row has not established a terminal # rescue state and must remain a barrier. for status in ("queued", "running", "exiting", "failed"): store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": status, })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict" def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation(): store = MemoryStore() registry = SimpleNamespace( components={"host-a": SimpleNamespace(store=store)}, ) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), registry=registry, ) store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict" store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", "reinstall_committed": True, })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") is None store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", "phase": "result_uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") is None # Rescue job uncertainty is never exempted by the operation id alone; # only the vm_rescue active/staged row proves its own terminal XML state. store.delete("reinstall_job", "job-1") store.set("rescue_job", "job-2", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict" store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "active", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None @pytest.mark.parametrize("identity_change", [ {"region": "sg"}, {"fabric_id": "fabric-sg"}, {"fabric_revision": 8}, {"cidr": "10.21.0.0/24"}, ]) def test_root_delete_rejects_catalog_identity_change_without_mutation( identity_change): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) asyncio.run(coordinator.apply(payload())) before = dict(store.values) async def must_not_mutate(*_args, **_kwargs): raise AssertionError("identity mismatch reached host mutation") coordinator._reconcile_dataplane = must_not_mutate fabric.get_status = lambda _fabric_id: (_ for _ in ()).throw( AssertionError("identity mismatch reached Fabric gate")) target = delete_payload( desired_revision=2, operation_id="vpc-delete-identity-mismatch-0001", **identity_change, ) with pytest.raises(VpcError) as error: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert error.value.status == 409 assert error.value.error_code == "vpc_delete_identity_mismatch" # Compare the store's exact encoded values: a rejected delete must not # rewrite timestamps, child projections, or the previous desired bytes. assert store.values == before def test_root_delete_without_previous_rejects_global_fabric_endpoint_overlap(): store = MemoryStore() conflicting = parse_fabric_plan({ "fabric_id": "fabric-sg", "region": "sg", "overlay_cidr": "100.64.2.0/24", "revision": 3, "nodes": [ {"host_id": "host-c", "overlay_ip": "100.64.2.1", "endpoint": "10.21.0.50:51820"}, {"host_id": "host-d", "overlay_ip": "100.64.2.2", "endpoint": "203.0.113.4:51820"}, ], }) save_fabric_catalog(store, conflicting) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) async def must_not_mutate(*_args, **_kwargs): raise AssertionError("conflicting delete reached host mutation") coordinator._reconcile_dataplane = must_not_mutate before = dict(store.values) target = delete_payload( vpc_id="vpc-never-applied", desired_revision=3, operation_id="vpc-delete-catalog-overlap-0001", cidr="10.21.0.0/24", ) with pytest.raises(VpcError) as error: asyncio.run(coordinator.delete("vpc-never-applied", target)) assert error.value.status == 409 assert error.value.error_code == "vpc_fabric_endpoint_overlap" assert store.values == before def test_root_delete_without_previous_fails_closed_on_stale_fabric_digest(): store = MemoryStore() fabric_plan = parse_fabric_plan({ "fabric_id": "fabric-sg", "region": "sg", "overlay_cidr": "100.64.2.0/24", "revision": 3, "nodes": [ {"host_id": "host-c", "overlay_ip": "100.64.2.1", "endpoint": "203.0.113.3:51820"}, {"host_id": "host-d", "overlay_ip": "100.64.2.2", "endpoint": "203.0.113.4:51820"}, ], }) save_fabric_catalog(store, fabric_plan, plan_digest="0" * 64) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) before = dict(store.values) target = delete_payload( vpc_id="vpc-never-applied", desired_revision=3, operation_id="vpc-delete-stale-fabric-0001", cidr="10.21.0.0/24", ) with pytest.raises(VpcError) as error: asyncio.run(coordinator.delete("vpc-never-applied", target)) assert error.value.status == 500 assert error.value.error_code == "vpc_fabric_record_corrupt" assert store.values == before def test_root_delete_accepts_revision_jump_and_superseded_unknown_host(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) asyncio.run(coordinator.apply(payload())) calls = [] async def confirm(plan, record, gate, previous_plan=None, **kwargs): calls.append({ "host_ids": sorted(host["host_id"] for host in record["hosts"]), "retiring": all(host.get("retiring") is True for host in record["hosts"]), "operation_id": kwargs.get("operation_id"), }) confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = confirm target = delete_payload( desired_revision=4, operation_id="vpc-delete-revision-jump-0001", hosts=("host-a", "host-b"), retiring_members=(retiring_member_a(),), ) result = asyncio.run(coordinator.delete("vpc-hk-1", target)) assert result == { "vpc_id": "vpc-hk-1", "operation_id": "vpc-delete-revision-jump-0001", "evidence_version": 2, "desired_revision": 4, "desired_digest": target["desired_digest"], "state": "deleted", "deleted": True, "readback": "confirmed", "removed_host_ids": ["host-a", "host-b"], "completed_at": result["completed_at"], } assert result["completed_at"] > 0 assert calls == [{ "host_ids": ["host-a", "host-b"], "retiring": True, "operation_id": "vpc-delete-revision-jump-0001", }] assert store.get("vpc_desired", "vpc-hk-1") is None assert store.list("vpc_host") == {} durable_receipt = json.loads(store.get( VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")) assert durable_receipt["readback"] == "confirmed" assert durable_receipt["completed_at"] == result["completed_at"] assert "retiring_members" not in durable_receipt # A receipt written by an earlier engine build is compacted on replay; # the digest retains the manifest commitment without permanently keeping # member UUID/private IP/MAC identifiers after successful deletion. legacy_receipt = dict(durable_receipt) legacy_receipt["retiring_members"] = target["retiring_members"] store.set(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1", json.dumps(legacy_receipt)) replay = asyncio.run(coordinator.delete("vpc-hk-1", target)) assert replay == result compacted_raw = store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") compacted = json.loads(compacted_raw) assert "retiring_members" not in compacted assert INSTANCE_A not in compacted_raw assert "10.20.0.10" not in compacted_raw assert retiring_member_a()["private_mac"] not in compacted_raw conflict = dict(target, operation_id="vpc-delete-revision-jump-0002") with pytest.raises(VpcError) as changed: asyncio.run(coordinator.delete("vpc-hk-1", conflict)) assert changed.value.status == 409 assert changed.value.error_code == "vpc_delete_receipt_conflict" with pytest.raises(VpcError) as recreated: asyncio.run(coordinator.apply(payload( desired_revision=5, operation_id="vpc-after-delete-0001"))) assert recreated.value.error_code == "vpc_deleted" def test_delete_manifest_accepts_unicast_with_odd_last_octet_only(): derived = derive_private_mac("never", "m") assert derived.endswith(":49") valid = delete_payload( vpc_id="never", cidr="10.24.0.0/24", desired_revision=1, operation_id="vpc-delete-mac-boundary-0001", hosts=("host-a",), retiring_members=({ "member_id": "m", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.24.0.10/32", "private_mac": derived, },), ) assert parse_vpc_delete_plan(valid)[1][0].private_mac == derived invalid = dict(valid) invalid.pop("desired_digest") invalid["retiring_members"] = [{ **valid["retiring_members"][0], "private_mac": "13:00:00:00:00:48", }] with pytest.raises(VpcError) as multicast: parse_vpc_delete_plan(invalid) assert multicast.value.error_code == "vpc_delete_manifest_invalid" invalid["retiring_members"][0]["private_mac"] = "00:00:00:00:00:00" with pytest.raises(VpcError) as all_zero: parse_vpc_delete_plan(invalid) assert all_zero.value.error_code == "vpc_delete_manifest_invalid" def test_delete_digest_matches_shared_php_fixture(): fixture = json.loads(( Path(__file__).with_name("fixtures") / "vpc_delete_digest.json" ).read_text(encoding="utf-8")) plan, retiring = parse_vpc_delete_plan(fixture["document"]) assert plan.desired_digest == fixture["expected_digest"] assert plan.desired_digest == ( "2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314" ) assert [host.host_id for host in plan.hosts] == ["11", "2"] assert [member.member_id for member in retiring] == ["1" * 32] def test_root_delete_recovers_required_host_after_restart(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) target = delete_payload( operation_id="vpc-delete-recover-0001", desired_revision=2, retiring_members=(retiring_member_a(),), ) async def interrupted(plan, record, gate, previous_plan=None, **kwargs): # Simulate a process loss after retiring-member cleanup consumed its # tombstone but before a host absence proof became durable. for host in record["hosts"]: host["retiring_members"] = [] coordinator._reconcile_dataplane = interrupted with pytest.raises(VpcError) as pending: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert pending.value.status == 503 assert pending.value.error_code == "vpc_delete_incomplete" durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["delete_target"]["required_host_ids"] == ["host-a"] assert durable["hosts"][0]["retiring_members"] == [] coordinator.recover() calls = [] async def confirm(plan, record, gate, previous_plan=None, **kwargs): calls.append({ "retiring": record["hosts"][0].get("retiring"), "retiring_members": list( record["hosts"][0].get("retiring_members", [])), }) confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = confirm result = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert result["deleted"] is True assert result["readback"] == "confirmed" assert calls == [{ "retiring": True, "retiring_members": [target["retiring_members"][0]], }] def test_root_delete_restart_retries_only_hosts_without_exact_removal_proof(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a", "host-b"))) asyncio.run(coordinator.apply(payload())) target = delete_payload( operation_id="vpc-delete-partial-restart-0001", desired_revision=2, hosts=("host-a", "host-b"), retiring_members=(retiring_member_a(),), ) calls = [] async def partial_then_confirm( plan, record, gate, previous_plan=None, **kwargs): host_ids = [host["host_id"] for host in record["hosts"]] calls.append(host_ids) if len(calls) == 1: assert host_ids == ["host-a", "host-b"] host_a = next( host for host in record["hosts"] if host["host_id"] == "host-a") confirm_delete_hosts(plan, {"hosts": [host_a]}) host_b = next( host for host in record["hosts"] if host["host_id"] == "host-b") host_b.update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": "vpc_host_unavailable", }) return assert host_ids == ["host-b"] confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = partial_then_confirm with pytest.raises(VpcError) as incomplete: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert incomplete.value.error_code == "vpc_delete_incomplete" durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) durable_hosts = {host["host_id"]: host for host in durable["hosts"]} assert coordinator._confirmed_removal( durable_hosts["host-a"], 2, target["desired_digest"]) assert not coordinator._confirmed_removal( durable_hosts["host-b"], 2, target["desired_digest"]) coordinator.recover() recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]} assert coordinator._confirmed_removal( recovered_hosts["host-a"], 2, target["desired_digest"]) assert recovered_hosts["host-b"]["state"] == "uncertain" result = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert calls == [["host-a", "host-b"], ["host-b"]] assert result["deleted"] is True assert result["removed_host_ids"] == ["host-a", "host-b"] assert result["readback"] == "confirmed" def test_delete_pending_projection_does_not_probe_confirmed_offline_host(): target = delete_payload( operation_id="vpc-delete-pending-probe-0001", desired_revision=2, hosts=("host-a", "host-b"), ) plan, _manifest = parse_vpc_delete_plan(target) nodes = ( FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820), FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820), ) fabric_plan = FabricPlan( "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, nodes, "f" * 64, ) probed = [] class RetryFabric(FakeFabric): @staticmethod def _plan_from_record(_record): return fabric_plan async def _components(self, retry_plan): for node in retry_plan.nodes: probed.append(node.host_id) if node.host_id == "host-a": raise FabricError( "confirmed host is now offline", 502, "fabric_host_capability_probe_failed", ) return { "host-b": ( SimpleNamespace(manager=object()), {"host_id": "host-b"}, ), } coordinator = VpcCoordinator( MemoryStore(), RetryFabric(hosts=("host-a", "host-b")), registry=object(), ) coordinator._validate_dataplane_resources = lambda _plan: None coordinator._fabric_record = lambda _plan: { "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, } coordinator._fabric_peers_for_plan = lambda _plan, _record: [] host_digests = [] def confirm_host(*args, **kwargs): host_digests.append(args[8]) return { "state": "active", "removed": True, "transport_verified": True, "private_dataplane_verified": True, "member_nics": [], "routes": [], "wg_allowed_ips": [], "proxy_ips": [], "bridge": None, "bridge_owner": None, "nft_table": None, "anti_spoof": False, "rp_filter": None, "proxy_arp": None, "proxy_arp_pvlan": None, "readback": "confirmed", "rollback": "none", } coordinator._run_host_dataplane = confirm_host record = { "vpc_id": plan.vpc_id, "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "hosts": [{ "host_id": "host-b", "member_ids": [], "retiring": True, "retiring_members": [], "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, }], } asyncio.run(coordinator._reconcile_dataplane_unlocked( plan, record, {"active": False, "available": True}, previous_plan=plan, operation_id=plan.operation_id, )) assert probed == ["host-b"] assert host_digests == [fabric_plan.digest] assert coordinator._confirmed_removal( record["hosts"][0], plan.desired_revision, plan.desired_digest) def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready(): target = payload( desired_revision=2, operation_id="vpc-op-config-proof-0002", hosts=[{"host_id": "host-a", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}], ) plan = parse_vpc_plan(target) fabric_plan = FabricPlan( "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, (FabricNode( "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), "f" * 64, ) class RuntimeFabric(FakeFabric): @staticmethod def _plan_from_record(_record): return fabric_plan async def _components(self, _plan): return { "host-a": ( SimpleNamespace(manager=object()), {"host_id": "host-a"}, ), } coordinator = VpcCoordinator( MemoryStore(), RuntimeFabric(hosts=("host-a",)), registry=object(), ) coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False coordinator._validate_dataplane_resources = lambda _plan: None coordinator._fabric_record = lambda _plan: { "public_keys": {"host-a": "a" * 44}, } coordinator._fabric_peers_for_plan = lambda _plan, _record: [] coordinator._run_host_dataplane = lambda *args, **kwargs: { "state": "applying", "transport_verified": True, "private_dataplane_verified": False, "config_verified": True, "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "observed_revision": 0, "observed_digest": None, "member_nics": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "config_verified": True, "xml_verified": True, "domain_running": False, "guest_network_verified": False, "restart_required": True, }], "reason_code": "vpc_private_guest_restart_required", } record = { "vpc_id": plan.vpc_id, "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "hosts": [{ "host_id": "host-a", "member_ids": ["member-b"], "retiring_members": [retiring_member_a()], "desired_revision": plan.desired_revision, "desired_digest": plan.desired_digest, }], } asyncio.run(coordinator._reconcile_dataplane_unlocked( plan, record, {"active": True, "available": True}, previous_plan=plan, )) assert record["hosts"][0]["state"] == "applying" assert record["hosts"][0]["retiring_members"] == [] def test_reconcile_skips_exact_host_so_later_host_gets_next_budget(): target = payload( hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ], ) plan = parse_vpc_plan(target) gate = {"active": True, "available": True} coordinator = VpcCoordinator( MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), registry=object(), ) record = coordinator._desired_record( plan, gate, state="applying") calls = [] def exact(host): member = next( item for item in plan.members if item.host_id == host["host_id"]) private_mac = derive_private_mac(plan.vpc_id, member.member_id) host.update({ "state": "active", "transport_verified": True, "private_dataplane_verified": True, "config_verified": True, "observed_revision": plan.desired_revision, "observed_digest": plan.desired_digest, "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], "member_nics": [{ "member_id": member.member_id, "instance_uuid": member.instance_uuid, "private_ip_cidr": f"{member.private_ip}/32", "private_mac": private_mac, "config_verified": True, "xml_verified": True, "seed_verified": True, "domain_running": True, "guest_network_verified": True, "restart_required": False, }], }) async def consume_one_host(_plan, execution, _gate, _previous, **kwargs): host_ids = [host["host_id"] for host in execution["hosts"]] calls.append(host_ids) if len(calls) == 1: exact(execution["hosts"][0]) execution["hosts"][1].update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "last_error_code": "vpc_private_guest_probe_deadline_exceeded", }) else: assert host_ids == ["host-b"] exact(execution["hosts"][0]) coordinator._reconcile_dataplane = consume_one_host asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) assert calls == [["host-a", "host-b"], ["host-b"]] assert all(host["state"] == "active" for host in record["hosts"]) def test_reconcile_reprobes_oldest_active_host_and_revokes_stale_guest_proof(): target = payload(hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) plan = parse_vpc_plan(target) gate = {"active": True, "available": True} coordinator = VpcCoordinator( MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), registry=object(), ) record = coordinator._desired_record(plan, gate, state="applying") attempted_at = {"host-a": 200, "host-b": 100} for host in record["hosts"]: member = next( item for item in plan.members if item.host_id == host["host_id"]) private_mac = derive_private_mac(plan.vpc_id, member.member_id) host.update({ "state": "active", "transport_verified": True, "private_dataplane_verified": True, "config_verified": True, "observed_revision": plan.desired_revision, "observed_digest": plan.desired_digest, "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "retiring_members": [], "probe_attempted_at_ns": attempted_at[host["host_id"]], "member_nics": [{ "member_id": member.member_id, "instance_uuid": member.instance_uuid, "private_ip_cidr": f"{member.private_ip}/32", "private_mac": private_mac, "config_verified": True, "xml_verified": True, "seed_verified": True, "domain_running": True, "guest_network_verified": True, "restart_required": False, }], }) calls = [] async def lose_qga_proof(_plan, execution, _gate, _previous, **kwargs): calls.append([host["host_id"] for host in execution["hosts"]]) host = execution["hosts"][0] host.update({ "state": "applying", "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": "vpc_private_guest_agent_unavailable", }) host["member_nics"][0].update({ "guest_network_verified": False, "guest_network_readback": "unavailable", "guest_reason_code": "vpc_private_guest_agent_unavailable", }) coordinator._reconcile_dataplane = lose_qga_proof asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) assert calls == [["host-b"]] hosts = {host["host_id"]: host for host in record["hosts"]} assert hosts["host-a"]["state"] == "active" assert hosts["host-b"]["state"] == "applying" assert hosts["host-b"]["private_dataplane_verified"] is False assert hosts["host-b"]["member_nics"][0][ "guest_network_verified"] is False assert record["state"] == "applying" assert record["next_stage_allowed"] is False assert record["private_dataplane_verified"] is False def test_reconcile_rotates_pending_host_after_it_consumes_budget(): target = payload(hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) plan = parse_vpc_plan(target) gate = {"active": True, "available": True} coordinator = VpcCoordinator( MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), registry=object(), ) record = coordinator._desired_record( plan, gate, state="applying") calls = [] async def consume_budget(_plan, execution, _gate, _previous, **kwargs): calls.append([host["host_id"] for host in execution["hosts"]]) consumer = execution["hosts"][0] consumer["probe_attempted_at_ns"] = 100 + len(calls) consumer.update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "last_error_code": "vpc_private_guest_probe_deadline_exceeded", }) coordinator._reconcile_dataplane = consume_budget asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan)) assert calls == [["host-a", "host-b"], ["host-b", "host-a"]] def test_instance_lifecycle_target_precedes_unrelated_pending_host(): target = payload(hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) plan = parse_vpc_plan(target) gate = {"active": True, "available": True} coordinator = VpcCoordinator( MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), registry=object(), ) record = coordinator._desired_record(plan, gate, state="applying") record["hosts"][1]["probe_attempted_at_ns"] = 999 calls = [] async def capture_order(_plan, execution, _gate, _previous, **kwargs): calls.append([host["host_id"] for host in execution["hosts"]]) coordinator._reconcile_dataplane = capture_order asyncio.run(coordinator._reconcile_pending_hosts( plan, record, gate, plan, private_link_states={INSTANCE_B: "down"}, )) assert calls == [["host-b", "host-a"]] def test_guest_probe_pool_is_rolling_bounded_and_deadline_fail_closed( monkeypatch): import sys import types try: import libvirt # noqa: F401 except ModuleNotFoundError: stub = types.ModuleType("libvirt") stub.libvirtError = type("libvirtError", (Exception,), {}) monkeypatch.setitem(sys.modules, "libvirt", stub) from virsh_panel.libvirt_service import VMService from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter members = [{ "member_id": f"member-{index}", "instance_uuid": f"00000000-0000-4000-8000-{index + 1:012x}", "private_ip_cidr": f"10.20.0.{index + 10}/32", "subnet_id": "subnet-a", } for index in range(10)] plan = parse_vpc_plan(payload(hosts=[{ "host_id": "host-a", "members": members, }])) monkeypatch.setattr("virsh_panel.vpc.VPC_GUEST_PROBE_WORKERS", 2) monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) def configure(self, instance_uuid, **kwargs): return { "xml": f"inactive:{instance_uuid}", "live": f"live:{instance_uuid}", "restart_required": False, } monkeypatch.setattr(VMService, "ensure_private_interface", configure) lock = threading.Lock() active = 0 maximum = 0 started = [] def probe(self, instance_uuid, **kwargs): nonlocal active, maximum assert isinstance(kwargs.get("guest_probe_deadline"), float) with lock: active += 1 maximum = max(maximum, active) started.append(instance_uuid) time.sleep(0.1) with lock: active -= 1 return { "inactive": f"inactive:{instance_uuid}", "live": f"live:{instance_uuid}", "guest_network": { "state": "confirmed", "verified": True, "source": "qemu_guest_agent", }, } monkeypatch.setattr(VMService, "read_private_domain_pair", probe) monkeypatch.setattr( VpcPrivateHostAdapter, "__init__", lambda self, manager, profile, **kwargs: None, ) monkeypatch.setattr( VpcPrivateHostAdapter, "apply", lambda self, dataplane: {}) captured = {} def observe(self, dataplane, *, xml_by_instance=None): captured.update(xml_by_instance or {}) return { "state": "applying", "config_verified": True, "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "member_nics": [], "reason_code": "vpc_private_guest_probe_deadline_exceeded", } monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe) coordinator = VpcCoordinator(MemoryStore(), FakeFabric()) result = coordinator._run_host_dataplane( plan, "host-a", SimpleNamespace(manager=object()), {"libvirt_uri": "qemu:///system", "storage_pool": "default", "iso_pool": "iso", "default_network": "default"}, [], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32, request_deadline=time.monotonic() + 1.05, ) assert maximum == 2 assert len(started) == 2 assert active == 0 assert len(captured) == 10 deadline_rows = [ pair["guest_network"] for pair in captured.values() if pair["guest_network"].get("reason_code") == "vpc_private_guest_probe_deadline_exceeded" ] assert len(deadline_rows) == 8 assert result["config_verified"] is True assert result["private_dataplane_verified"] is False def test_guest_probe_default_pool_bounds_libvirt_reply_amplification(): from virsh_panel.vpc import VPC_GUEST_PROBE_WORKERS # virDomainQemuAgentCommand materializes libvirt's bounded raw reply before # Python can enforce its 192 KiB parse limit. Keep the default narrow even # though individual tests may monkeypatch it lower for deterministic timing. assert VPC_GUEST_PROBE_WORKERS == 4 def test_guest_probe_limit_is_process_wide_across_vpc_requests(monkeypatch): import sys import types try: import libvirt # noqa: F401 except ModuleNotFoundError: stub = types.ModuleType("libvirt") stub.libvirtError = type("libvirtError", (Exception,), {}) monkeypatch.setitem(sys.modules, "libvirt", stub) from virsh_panel.libvirt_service import VMService from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter def make_plan(label, second_octet, first_uuid): members = [{ "member_id": f"member-{label}-{index}", "instance_uuid": f"00000000-0000-4000-8000-{first_uuid + index:012x}", "private_ip_cidr": f"10.{second_octet}.0.{index + 10}/32", "subnet_id": f"subnet-{label}", } for index in range(4)] return parse_vpc_plan(payload( vpc_id=f"vpc-hk-{label}", cidr=f"10.{second_octet}.0.0/24", operation_id=f"vpc-op-{label}-00000001", subnets=[{ "subnet_id": f"subnet-{label}", "cidr": f"10.{second_octet}.0.0/25", }], hosts=[{"host_id": "host-a", "members": members}], )) plans = (make_plan("a", 20, 1), make_plan("b", 21, 101)) monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) monkeypatch.setattr( VMService, "ensure_private_interface", lambda self, instance_uuid, **kwargs: { "xml": f"inactive:{instance_uuid}", "live": f"live:{instance_uuid}", "restart_required": False, }, ) lock = threading.Lock() release_full_wave = threading.Event() active = 0 maximum = 0 started = 0 def probe(self, instance_uuid, **kwargs): nonlocal active, maximum, started with lock: active += 1 started += 1 maximum = max(maximum, active) if active >= 8: release_full_wave.set() # Without the process gate, both four-worker request pools enter # together. With it, a bounded wave of four leaves before the next. release_full_wave.wait(0.2) with lock: active -= 1 return { "inactive": f"inactive:{instance_uuid}", "live": f"live:{instance_uuid}", "guest_network": { "state": "confirmed", "verified": True, "source": "qemu_guest_agent", }, } monkeypatch.setattr(VMService, "read_private_domain_pair", probe) monkeypatch.setattr( VpcPrivateHostAdapter, "__init__", lambda self, manager, profile, **kwargs: None, ) monkeypatch.setattr( VpcPrivateHostAdapter, "apply", lambda self, dataplane: {}) def observe(self, dataplane, *, xml_by_instance=None): return { "state": "active", "config_verified": True, "config_observed_revision": dataplane.vpc.desired_revision, "config_observed_digest": dataplane.vpc.desired_digest, "member_nics": [], } monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe) def run(plan): return VpcCoordinator( MemoryStore(), FakeFabric(), )._run_host_dataplane( plan, "host-a", SimpleNamespace(manager=object()), {"libvirt_uri": "qemu:///system", "storage_pool": "default", "iso_pool": "iso", "default_network": "default"}, [], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32, request_deadline=time.monotonic() + 3, ) with ThreadPoolExecutor(max_workers=2) as requests: results = [future.result() for future in ( requests.submit(run, plans[0]), requests.submit(run, plans[1]), )] assert started == 8 assert maximum == 4 assert all(result["config_verified"] is True for result in results) def test_request_probe_deadline_stops_next_host_and_keeps_config_proof( monkeypatch): target = payload(hosts=[ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", }]}, ]) plan = parse_vpc_plan(target) fabric_plan = FabricPlan( "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, ( FabricNode( "host-a", "100.64.0.1", "192.0.2.1:51820", 51820), FabricNode( "host-b", "100.64.0.2", "192.0.2.2:51820", 51820), ), "f" * 64, ) class DeadlineFabric(FakeFabric): @staticmethod def _plan_from_record(_record): return fabric_plan async def _components(self, _plan): return { host_id: ( SimpleNamespace(manager=object()), {"host_id": host_id}, ) for host_id in ("host-a", "host-b") } coordinator = VpcCoordinator( MemoryStore(), DeadlineFabric(hosts=("host-a", "host-b")), registry=object(), ) coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False coordinator._validate_dataplane_resources = lambda _plan: None coordinator._fabric_record = lambda _plan: { "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, } coordinator._fabric_peers_for_plan = lambda _plan, _record: [] monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.05) calls = [] def slow_host(*args, **kwargs): host_id = args[1] calls.append(host_id) time.sleep(0.08) return { "state": "active", "transport_verified": True, "private_dataplane_verified": True, "config_verified": True, "config_observed_revision": plan.desired_revision, "config_observed_digest": plan.desired_digest, "member_nics": [{ "member_id": "member-a", "guest_network_verified": True, }], "reason_code": None, } coordinator._run_host_dataplane = slow_host gate = {"active": True, "available": True} record = coordinator._desired_record( plan, gate, state="applying") asyncio.run(coordinator._reconcile_dataplane_unlocked( plan, record, gate, previous_plan=plan)) assert calls == ["host-a"] first, second = record["hosts"] assert first["transport_verified"] is True assert first["probe_attempted_at_ns"] > 0 assert first["config_verified"] is True assert first["config_observed_revision"] == plan.desired_revision assert first["private_dataplane_verified"] is False assert first["member_nics"][0]["guest_network_verified"] is False assert first["last_error_code"] \ == "vpc_private_guest_probe_deadline_exceeded" assert second["transport_verified"] is False assert second["last_error_code"] \ == "vpc_private_guest_probe_deadline_exceeded" def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors(): deterministic_codes = ( "vpc_private_remove_requires_stopped", "vpc_private_remove_unsafe", "vpc_private_interface_conflict", "vpc_seed_missing", "vpc_seed_readback_invalid", "vpc_private_xml_invalid", ) for index, error_code in enumerate(deterministic_codes, start=1): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) target = delete_payload( operation_id=f"vpc-delete-deterministic-{index:04d}", retiring_members=(retiring_member_a(),), ) async def blocked(plan, record, gate, previous_plan=None, **kwargs): record["hosts"][0].update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": error_code, }) coordinator._reconcile_dataplane = blocked with pytest.raises(VpcError) as blocked_delete: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert blocked_delete.value.status == 409 assert blocked_delete.value.error_code == error_code assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) target = delete_payload( operation_id="vpc-delete-transient-error-0001", retiring_members=(retiring_member_a(),), ) async def transient(plan, record, gate, previous_plan=None, **kwargs): record["hosts"][0].update({ "state": "uncertain", "transport_verified": False, "private_dataplane_verified": False, "observed_revision": 0, "observed_digest": None, "last_error_code": "vpc_private_remove_uncertain", }) coordinator._reconcile_dataplane = transient with pytest.raises(VpcError) as transient_delete: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert transient_delete.value.status == 503 assert transient_delete.value.error_code == "vpc_delete_incomplete" @pytest.mark.parametrize("include_proven_host", [False, True]) def test_root_delete_carries_forward_exact_removed_host_without_recontact( include_proven_host): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) detached = payload( desired_revision=2, operation_id="vpc-member-detach-0001", subnets=[], members=[], routes=[], acls=[], hosts=[], ) # Ordinary apply models the earlier member/deployment detach. Its host # tombstone is then completed with the exact revision/digest proof that a # lost PHP response may leave behind. asyncio.run(coordinator.apply(detached)) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["hosts"][0]["retiring"] is True confirm_delete_hosts(parse_vpc_plan(detached), durable) store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) async def must_not_recontact(*args, **kwargs): raise AssertionError("exact removed host must not be contacted again") coordinator._reconcile_dataplane = must_not_recontact target = delete_payload( desired_revision=3, operation_id=("vpc-delete-proven-included-0001" if include_proven_host else "vpc-delete-proven-omitted-0001"), hosts=(("host-a",) if include_proven_host else ()), ) result = asyncio.run(coordinator.delete("vpc-hk-1", target)) assert result["removed_host_ids"] == ["host-a"] assert result["readback"] == "confirmed" def test_root_delete_requires_every_unproven_old_host_in_target(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) target = delete_payload( desired_revision=2, operation_id="vpc-delete-missing-host-0001", hosts=(), ) with pytest.raises(VpcError) as missing: asyncio.run(coordinator.delete("vpc-hk-1", target)) assert missing.value.status == 409 assert missing.value.error_code == "vpc_delete_host_set_mismatch" assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None def test_root_delete_without_desired_record_proves_requested_hosts_or_empty(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) calls = [] async def confirm(plan, record, gate, previous_plan=None, **kwargs): calls.append(( [host["host_id"] for host in record["hosts"]], [host.get("retiring") for host in record["hosts"]], [row["instance_uuid"] for host in record["hosts"] for row in host.get("retiring_members", [])], )) confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = confirm target = delete_payload( vpc_id="vpc-never-applied", operation_id="vpc-delete-never-applied-0001", desired_revision=3, hosts=("host-a",), cidr="10.21.0.0/24", retiring_members=({ "member_id": "member-never-applied", "instance_uuid": INSTANCE_B, "host_id": "host-a", "private_ip_cidr": "10.21.0.10/32", "private_mac": derive_private_mac( "vpc-never-applied", "member-never-applied"), },), ) result = asyncio.run(coordinator.delete("vpc-never-applied", target)) assert result["removed_host_ids"] == ["host-a"] assert calls == [(["host-a"], [True], [INSTANCE_B])] empty = delete_payload( vpc_id="vpc-empty-never-applied", operation_id="vpc-delete-empty-never-applied-0001", desired_revision=1, hosts=(), cidr="10.22.0.0/24", ) result = asyncio.run(coordinator.delete( "vpc-empty-never-applied", empty)) assert result["removed_host_ids"] == [] assert len(calls) == 1 def test_delete_manifest_removes_unknown_member_nic_before_host_resources( monkeypatch): import sys import types try: import libvirt # noqa: F401 except ModuleNotFoundError: stub = types.ModuleType("libvirt") stub.libvirtError = type("libvirtError", (Exception,), {}) monkeypatch.setitem(sys.modules, "libvirt", stub) from virsh_panel.libvirt_service import VMService from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter target = delete_payload( vpc_id="vpc-remote-only", operation_id="vpc-delete-remote-only-0001", desired_revision=5, hosts=("host-a",), cidr="10.23.0.0/24", retiring_members=({ "member_id": "member-remote-only", "instance_uuid": INSTANCE_B, "host_id": "host-a", "private_ip_cidr": "10.23.0.10/32", "private_mac": derive_private_mac( "vpc-remote-only", "member-remote-only"), },), ) plan, manifest = parse_vpc_delete_plan(target) rows = [{ "member_id": member.member_id, "instance_uuid": member.instance_uuid, "host_id": member.host_id, "private_ip_cidr": f"{member.private_ip}/32", "private_mac": member.private_mac, } for member in manifest] events = [] monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) def remove_nic(self, instance_uuid, **kwargs): events.append(("nic", instance_uuid, kwargs["vpc_id"])) return {"removed": True, "readback": "confirmed"} monkeypatch.setattr(VMService, "remove_private_interface", remove_nic) monkeypatch.setattr( VpcPrivateHostAdapter, "__init__", lambda self, manager, profile, **kwargs: None, ) def remove_host(self, dataplane): events.append(("host", dataplane.host_id, dataplane.vpc.vpc_id)) return { "state": "active", "removed": True, "bridge": None, "bridge_owner": None, "nft_table": None, "anti_spoof": False, "rp_filter": None, "proxy_arp": None, "proxy_arp_pvlan": None, "proxy_ips": [], "member_nics": [], "routes": [], "wg_allowed_ips": [], "readback": "confirmed", } monkeypatch.setattr(VpcPrivateHostAdapter, "remove", remove_host) coordinator = VpcCoordinator(MemoryStore(), FakeFabric()) observed = coordinator._run_host_dataplane( plan, "host-a", SimpleNamespace(manager=object()), {"libvirt_uri": "qemu:///system", "storage_pool": "default", "iso_pool": "iso", "default_network": "default"}, [], None, retiring_members=rows, fabric_public_keys={}, fabric_plan_digest="ef" * 32, ) assert events == [ ("nic", INSTANCE_B, "vpc-remote-only"), ("host", "host-a", "vpc-remote-only"), ] assert observed["removed"] is True assert observed["readback"] == "confirmed" def test_root_delete_receipt_survives_purge_failure_and_keeps_timestamp(): store = MemoryStore() coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def confirm(plan, record, gate, previous_plan=None, **kwargs): confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = confirm target = delete_payload( operation_id="vpc-delete-purge-crash-0001", retiring_members=(retiring_member_a(),), ) original_delete = store.delete failed = False def fail_once(kind, key): nonlocal failed if kind == "vpc_desired" and not failed: failed = True raise RuntimeError("crash after receipt commit") return original_delete(kind, key) store.delete = fail_once with pytest.raises(RuntimeError, match="after receipt commit"): asyncio.run(coordinator.delete("vpc-hk-1", target)) durable_receipt = json.loads(store.get( VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")) first_completed_at = durable_receipt["completed_at"] assert first_completed_at > 0 result = asyncio.run(coordinator.delete("vpc-hk-1", target)) assert result["completed_at"] == first_completed_at assert result["readback"] == "confirmed" assert store.get("vpc_desired", "vpc-hk-1") is None