"""Structured VPC control-plane tests (no host data-plane commands).""" from __future__ import annotations import asyncio import json import threading from contextlib import asynccontextmanager from types import SimpleNamespace import pytest from virsh_panel.fabric import FabricError from virsh_panel.vpc import ( VPC_SCHEMA_VERSION, VpcCoordinator, VpcError, _plan_from_record, parse_vpc_plan, ) from virsh_panel.vpc_dataplane import derive_private_mac INSTANCE_A = "11111111-1111-4111-8111-111111111111" INSTANCE_B = "22222222-2222-4222-8222-222222222222" class MemoryStore: def __init__(self): self.values = {} self.lock = threading.RLock() def get(self, kind, key): with self.lock: return self.values.get((kind, key)) def set(self, kind, key, value): with self.lock: self.values[kind, key] = value def delete(self, kind, key): with self.lock: self.values.pop((kind, key), None) def list(self, kind): with self.lock: return {key: value for (row_kind, key), value in self.values.items() if row_kind == kind} class FakeFabric: def __init__(self, *, state="active", gate=True, revision=7, hosts=("host-a", "host-b")): self.state = state self.gate = gate self.revision = revision self.hosts = hosts def get_status(self, fabric_id): if fabric_id == "missing": raise FabricError("missing", 404, "fabric_not_found") return { "fabric_id": fabric_id, "region": "hk", "revision": self.revision, "state": self.state, "next_stage_allowed": self.gate, "nodes": [{"host_id": host} for host in self.hosts], } def payload(**overrides): value = { "schema_version": VPC_SCHEMA_VERSION, "vpc_id": "vpc-hk-1", "region": "hk", "fabric_id": "fabric-hk", "fabric_revision": 7, "cidr": "10.20.0.0/24", "desired_revision": 1, "operation_id": "vpc-op-00000001", "subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}], "hosts": [{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}], "routes": [], "acls": [], } value.update(overrides) return value def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state(): first = parse_vpc_plan(payload()) second = parse_vpc_plan(payload( hosts=None, members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.10", "subnet_id": "subnet-a", }], )) assert first.desired_digest == second.desired_digest assert first.members[0].private_ip == "10.20.0.10" with pytest.raises(VpcError): parse_vpc_plan(payload(cidr="192.0.2.0/24")) with pytest.raises(VpcError): parse_vpc_plan(payload(cidr="10.20.0.0/24", network_cidr="10.21.0.0/24")) with pytest.raises(VpcError): parse_vpc_plan(payload(subnets=[ {"subnet_id": "a", "cidr": "10.20.0.0/25"}, {"subnet_id": "b", "cidr": "10.20.0.64/26"}, ])) with pytest.raises(VpcError): parse_vpc_plan(payload(acls=[{ "acl_id": "public", "direction": "ingress", "action": "allow", "protocol": "tcp", "source_cidr": "0.0.0.0/0", "destination_cidr": "10.20.0.10/32", "priority": 1, }])) def test_member_uuid_matches_existing_libvirt_identity_boundary(): # Existing panel/libvirt records only require the canonical UUID shape; # operator-supplied UUIDs may use non-RFC version/variant nibbles. The # engine must accept those persisted VM identities in a VPC DTO too. legacy_uuid = "11111111-1111-6111-0111-111111111111" parsed = parse_vpc_plan(payload( hosts=[{"host_id": "host-a", "members": ["member-a"]}], members=[{ "member_id": "member-a", "instance_uuid": legacy_uuid, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], )) assert parsed.members[0].instance_uuid == legacy_uuid def test_digest_matches_php_wire_shape_for_host_member_grouping(): wire = payload( hosts=[{"host_id": "host-a", "members": ["member-a"]}], members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], ) # PHP computes desired_digest before attaching operation_id to the # signed/replayed payload; operation_id is an envelope field, not part of # desired document identity. digest_wire = {key: value for key, value in wire.items() if key != "operation_id"} expected = __import__("hashlib").sha256( json.dumps(digest_wire, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") ).hexdigest() parsed = parse_vpc_plan(wire) assert parsed.desired_digest == expected assert parsed.canonical()["hosts"] == [{ "host_id": "host-a", "members": ["member-a"] }] def test_plan_validates_routes_acl_ports_and_member_scope(): parsed = parse_vpc_plan(payload( hosts=[ {"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}, {"host_id": "host-b", "members": [ {"member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"}, ]}, ], routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32", "next_hop_member_id": "member-a", "metric": 20}], acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow", "protocol": "tcp", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.10/32", "port_from": 443, "port_to": 443, "priority": 1}], )) assert parsed.host_ids == {"host-a", "host-b"} assert parsed.routes[0].next_hop_member_id == "member-a" exact = parse_vpc_plan(payload(routes=[{ "route_id": "exact", "destination_cidr": "10.20.0.10/32", "next_hop_member_id": "member-a", }])) assert exact.routes[0].destination_cidr == "10.20.0.10/32" high_priority = parse_vpc_plan(payload(acls=[{ "acl_id": "high", "direction": "ingress", "action": "allow", "protocol": "any", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.10/32", "priority": 1_000_000, }])) assert high_priority.acls[0].priority == 1_000_000 with pytest.raises(VpcError): parse_vpc_plan(payload(routes=[{ "route_id": "bad", "destination_cidr": "0.0.0.0/0", "next_hop_host_id": "host-a", }])) with pytest.raises(VpcError) as broad_route: parse_vpc_plan(payload(routes=[{ "route_id": "broad", "destination_cidr": "10.20.0.128/25", "next_hop_member_id": "member-a", }])) assert broad_route.value.error_code == "vpc_route_not_exact" with pytest.raises(VpcError): parse_vpc_plan(payload(acls=[{ "acl_id": "bad", "direction": "egress", "action": "allow", "protocol": "icmp", "source_cidr": "10.20.0.0/24", "destination_cidr": "10.20.0.0/24", "port_from": 1, }])) with pytest.raises(VpcError): parse_vpc_plan(payload(routes=[{ "route_id": "alias-conflict", "destination_cidr": "10.20.0.0/25", "destination": "10.20.0.128/25", "next_hop_member_id": "member-a", }])) with pytest.raises(VpcError): parse_vpc_plan(payload(hosts=[{ "host_id": "host-a", "members": ["member-a", "member-a"], }])) with pytest.raises(VpcError): parse_vpc_plan(payload( hosts=[], members=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], )) def test_coordinator_persists_desired_children_and_blocks_without_fabric(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False)) result = asyncio.run(coordinator.apply(payload())) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["observed_revision"] == 0 assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"} assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"} assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"} def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral(): store = MemoryStore() result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) assert result["state"] == "applying" assert result["next_stage_allowed"] is False assert result["observed"]["fabric_gate"]["active"] is True assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True def test_recovery_reopens_only_the_control_plane_as_applying(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) asyncio.run(coordinator.apply(payload())) coordinator.recover() recovered = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert recovered["state"] == "applying" assert recovered["next_stage_allowed"] is False assert recovered["hosts"][0]["state"] == "applying" assert recovered["hosts"][0]["transport_verified"] is True def test_empty_vpc_cannot_open_fabric_gate(): store = MemoryStore() result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload( hosts=[], members=[], ))) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts" result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) assert result["state"] == "degraded" assert result["fabric_gate"]["error_code"] == "vpc_no_members" def test_coordinator_requires_exact_fabric_revision_and_all_host_readback(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a", "host-b")) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload( hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}], ))) assert result["state"] == "applying" raw = json.loads(store.get("vpc_desired", "vpc-hk-1")) digest = raw["desired_digest"] observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True, "readback": {"safe": "ok", "oversized": "x" * 513}}, })) assert observed["state"] == "active" assert observed["next_stage_allowed"] is True assert observed["hosts"][0]["readback"] == {"safe": "ok"} fabric.revision = 8 blocked = asyncio.run(coordinator.reconcile("vpc-hk-1")) assert blocked["state"] == "degraded" assert blocked["next_stage_allowed"] is False def test_transport_only_observation_never_opens_private_dataplane_gate(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload())) digest = result["desired_digest"] pending = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True}, })) assert pending["state"] == "applying" assert pending["next_stage_allowed"] is False assert pending["private_dataplane_verified"] is False assert pending["hosts"][0]["state"] == "applying" assert pending["hosts"][0]["last_error_code"] == ( "vpc_private_dataplane_unverified" ) def test_partial_host_observation_is_degraded_and_never_active(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a", "host-b")) coordinator = VpcCoordinator(store, fabric) result = asyncio.run(coordinator.apply(payload( hosts=[ {"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"}, ]}, {"host_id": "host-b", "members": [ {"member_id": "member-b", "instance_uuid": INSTANCE_B, "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"}, ]}, ], ))) digest = result["desired_digest"] observed = asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) assert observed["state"] == "degraded" assert observed["next_stage_allowed"] is False with pytest.raises(VpcError) as inconsistent: asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": False, "private_dataplane_verified": True}, "host-b": {"state": "degraded", "observed_revision": 0, "observed_digest": None, "transport_verified": False}, })) assert inconsistent.value.error_code == "vpc_observation_invalid" def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active(): store = MemoryStore() fabric = FakeFabric() original = fabric.get_status def wrong_region(fabric_id): status = original(fabric_id) status["region"] = "us" return status fabric.get_status = wrong_region result = asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["fabric_gate"]["error_code"] == "vpc_fabric_region_mismatch" @pytest.mark.parametrize("mutation", [ lambda status: status.update(fabric_id="other-fabric"), lambda status: status.update(revision=True), ]) def test_fabric_gate_rejects_tampered_identity_or_revision(mutation): store = MemoryStore() fabric = FakeFabric() original = fabric.get_status def tampered(fabric_id): status = original(fabric_id) mutation(status) return status fabric.get_status = tampered result = asyncio.run(VpcCoordinator(store, fabric).apply(payload())) assert result["next_stage_allowed"] is False assert result["state"] == "degraded" def test_coordinator_revision_conflict_and_tampered_record_fail_closed(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) first = asyncio.run(coordinator.apply(payload())) replay = asyncio.run(coordinator.apply(payload())) assert replay["desired_digest"] == first["desired_digest"] with pytest.raises(VpcError) as error: asyncio.run(coordinator.apply(payload( operation_id="vpc-op-00000002", members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}], hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}, ]}], ))) assert error.value.error_code == "vpc_revision_conflict" with pytest.raises(VpcError) as invalid_revision: asyncio.run(coordinator.apply(payload( desired_revision=0, operation_id="vpc-op-00000003", ))) assert invalid_revision.value.error_code == "vpc_invalid" asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000004", ))) with pytest.raises(VpcError) as stale: asyncio.run(coordinator.apply(payload( desired_revision=1, operation_id="vpc-op-00000005", members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A, "host_id": "host-a", "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}], hosts=[{"host_id": "host-a", "members": [ {"member_id": "member-a", "instance_uuid": INSTANCE_A, "private_ip": "10.20.0.12", "subnet_id": "subnet-a"}, ]}], ))) assert stale.value.error_code == "vpc_revision_conflict" tampered = json.loads(store.get("vpc_desired", "vpc-hk-1")) tampered["desired_digest"] = "0" * 64 store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered)) with pytest.raises(VpcError) as corrupt: asyncio.run(coordinator.reconcile("vpc-hk-1")) assert corrupt.value.error_code == "vpc_record_corrupt" tampered["vpc_id"] = "other-vpc" tampered["desired_digest"] = first["desired_digest"] store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered)) with pytest.raises(VpcError) as mismatched: coordinator.get_status("vpc-hk-1") assert mismatched.value.error_code == "vpc_record_corrupt" # Restore a structurally valid root and corrupt only observed evidence; # a newer target must not overwrite that evidence silently. restored = json.loads(store.get("vpc_desired", "vpc-hk-1")) restored["vpc_id"] = "vpc-hk-1" restored["desired_revision"] = 2 restored["desired_digest"] = parse_vpc_plan(payload( desired_revision=2, operation_id="vpc-op-00000004", )).desired_digest restored["observed_revision"] = 99 store.set("vpc_desired", "vpc-hk-1", json.dumps(restored)) with pytest.raises(VpcError) as observed_corrupt: asyncio.run(coordinator.apply(payload( desired_revision=3, operation_id="vpc-op-00000006", ))) assert observed_corrupt.value.error_code == "vpc_record_corrupt" def test_recover_invalidates_observed_gate_and_record_round_trips(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) raw = json.loads(store.get("vpc_desired", "vpc-hk-1")) raw["state"] = "active" raw["next_stage_allowed"] = True raw["observed_revision"] = 1 raw["observed_digest"] = raw["desired_digest"] raw["hosts"][0].update(state="active", observed_revision=1, observed_digest=raw["desired_digest"], transport_verified=True, private_dataplane_verified=True) store.set("vpc_desired", "vpc-hk-1", json.dumps(raw)) coordinator.recover() recovered = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert recovered["state"] == "uncertain" assert recovered["next_stage_allowed"] is False assert recovered["observed_revision"] == 1 assert recovered["observed_digest"] == recovered["desired_digest"] assert recovered["fabric_gate"]["active"] is False assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending" assert recovered["hosts"][0]["observed_revision"] == 1 assert _plan_from_record(recovered).vpc_id == "vpc-hk-1" def test_removed_children_are_tombstoned_until_observed_cleanup(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) changed = payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ) asyncio.run(coordinator.apply(changed)) old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a")) assert old["state"] == "removed" assert old["removed"] is True def test_new_desired_revision_keeps_last_observed_evidence_until_readback(): store = MemoryStore() fabric = FakeFabric(hosts=("host-a",)) coordinator = VpcCoordinator(store, fabric) first = asyncio.run(coordinator.apply(payload())) digest = first["desired_digest"] asyncio.run(coordinator.observe("vpc-hk-1", { "host-a": {"state": "active", "observed_revision": 1, "observed_digest": digest, "transport_verified": True, "private_dataplane_verified": True}, })) second = asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", ))) assert second["state"] == "applying" assert second["next_stage_allowed"] is False assert second["observed_revision"] == 1 assert second["observed_digest"] == digest assert second["hosts"][0]["observed_revision"] == 1 def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): host = record["hosts"][0] requested = (private_link_states or {}).get(INSTANCE_A, "preserve") host.update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": derive_private_mac("vpc-hk-1", "member-a"), "xml_verified": True, "link_state": "down" if requested == "down" else "up", }], ) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="down", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert state["readback"] == "confirmed" assert state["state"] == "suspended" assert state["vpc_id"] == "vpc-hk-1" assert state["member_id"] == "member-a" assert state["link_state"] == "down" assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a") # Preserve does not implicitly request link-up from the coordinator. state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="preserve", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, )) assert state["link_state"] == "up" def test_removed_member_reconcile_uses_durable_retiring_tombstone(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): host = record["hosts"][0] host.update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, member_nics=[], retiring_members=[], ) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="absent", expected={"vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_state": "removed"}, )) assert state["state"] == "removed" assert state["readback"] == "confirmed" assert state["link_state"] == "absent" def test_instance_reconcile_rejects_stale_revision_before_host_apply(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) calls = [] async def fake_reconcile(*args, **kwargs): calls.append((args, kwargs)) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={ "required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_revision": 99, "private_mac": derive_private_mac("vpc-hk-1", "member-a"), }, )) assert calls == [] assert state["readback"] == "missing" assert state["reason_code"] == "vpc_private_expectation_mismatch" def test_instance_reconcile_rejects_host_mac_drift(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) asyncio.run(coordinator.apply(payload())) async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): record["hosts"][0].update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": "02:00:00:00:00:01", "xml_verified": True, "link_state": "up", }], ) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", expected={"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "private_mac": derive_private_mac("vpc-hk-1", "member-a")}, )) assert state["readback"] == "missing" assert state["state"] == "uncertain" def test_removed_member_reconcile_locks_retiring_vm_identity(): store = MemoryStore() acquired = [] @asynccontextmanager async def vm_lock(host_id, instance_uuid): acquired.append((host_id, instance_uuid)) yield coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock) asyncio.run(coordinator.apply(payload())) asyncio.run(coordinator.apply(payload( desired_revision=2, operation_id="vpc-op-00000002", hosts=[{"host_id": "host-a", "members": []}], members=[], ))) durable = json.loads(store.get("vpc_desired", "vpc-hk-1")) # One host pass removes every retiring member. Keep a sibling tombstone # in the durable row to prove the lock set is not limited to the request # target. durable["hosts"][0]["retiring_members"].append({ "member_id": "member-b", "instance_uuid": INSTANCE_B, "host_id": "host-a", "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", "private_mac": derive_private_mac("vpc-hk-1", "member-b"), }) store.set("vpc_desired", "vpc-hk-1", json.dumps(durable)) acquired.clear() async def fake_unlocked(plan, record, gate, previous_plan=None, private_link_states=None): host = record["hosts"][0] host.update( state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, member_nics=[], retiring_members=[], ) coordinator._reconcile_dataplane_unlocked = fake_unlocked state = asyncio.run(coordinator.reconcile_instance( INSTANCE_A, host_id="host-a", link_state="absent", expected={"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a", "desired_state": "removed", "private_mac": derive_private_mac("vpc-hk-1", "member-a")}, )) assert ("host-a", INSTANCE_A) in acquired assert ("host-a", INSTANCE_B) in acquired assert state["state"] == "removed" assert state["readback"] == "confirmed" def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation(): store = MemoryStore() registry = SimpleNamespace( components={"host-a": SimpleNamespace(store=store)}, ) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), registry=registry, ) key = f"host-scope.{INSTANCE_A}" store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "active", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict" store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "staged", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None # A same-operation queued/failed row has not established a terminal # rescue state and must remain a barrier. for status in ("queued", "running", "exiting", "failed"): store.set("vm_rescue", key, json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": status, })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict" def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation(): store = MemoryStore() registry = SimpleNamespace( components={"host-a": SimpleNamespace(store=store)}, ) coordinator = VpcCoordinator( store, FakeFabric(hosts=("host-a",)), registry=registry, ) store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict" store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", "reinstall_committed": True, })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") is None store.set("reinstall_job", "job-1", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "reinstall-op-1", "status": "uncertain", "phase": "result_uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "reinstall-op-1") is None # Rescue job uncertainty is never exempted by the operation id alone; # only the vm_rescue active/staged row proves its own terminal XML state. store.delete("reinstall_job", "job-1") store.set("rescue_job", "job-2", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "uncertain", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict" store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({ "vm_id": INSTANCE_A, "operation_id": "rescue-op-1", "status": "active", })) assert coordinator._lifecycle_conflict( "host-a", INSTANCE_A, "rescue-op-1") is None