You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc.py

4226 lines
158 KiB
Python

"""Structured VPC control-plane tests (no host data-plane commands)."""
from __future__ import annotations
import asyncio
from concurrent.futures import ThreadPoolExecutor
import json
import threading
import time
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
import pytest
from virsh_panel.fabric import (
FabricCoordinator,
FabricError,
FabricNode,
FabricPlan,
parse_fabric_plan,
)
from virsh_panel.vpc import (
VPC_DELETE_RECEIPT_KIND,
VPC_GUEST_EXEC_KIND,
VPC_SCHEMA_VERSION,
VpcCoordinator,
VpcError,
_plan_from_record,
parse_vpc_delete_plan,
parse_vpc_plan,
)
from virsh_panel.vpc_dataplane import derive_private_mac
INSTANCE_A = "11111111-1111-4111-8111-111111111111"
INSTANCE_B = "22222222-2222-4222-8222-222222222222"
class MemoryStore:
def __init__(self):
self.values = {}
self.lock = threading.RLock()
def get(self, kind, key):
with self.lock:
return self.values.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.values[kind, key] = value
def delete(self, kind, key):
with self.lock:
self.values.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.values.items()
if row_kind == kind}
class FakeFabric:
def __init__(self, *, state="active", gate=True, revision=7,
hosts=("host-a", "host-b"), endpoint_ips=None):
self.state = state
self.gate = gate
self.revision = revision
self.hosts = hosts
self.endpoint_ips = endpoint_ips or tuple(
f"203.0.113.{index + 1}" for index, _host in enumerate(hosts))
def get_status(self, fabric_id):
if fabric_id == "missing":
raise FabricError("missing", 404, "fabric_not_found")
return {
"fabric_id": fabric_id,
"region": "hk",
"revision": self.revision,
"state": self.state,
"next_stage_allowed": self.gate,
"nodes": [
{"host_id": host,
"endpoint": f"{self.endpoint_ips[index]}:51820"}
for index, host in enumerate(self.hosts)
],
}
def test_reconcile_loop_preserves_and_coalesces_wakeups_during_a_pass():
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
async def exercise():
entered = asyncio.Event()
release = asyncio.Event()
completed = asyncio.Queue()
calls = 0
async def reconcile_all():
nonlocal calls
calls += 1
if calls == 1:
entered.set()
await release.wait()
completed.put_nowait(calls)
coordinator.reconcile_all = reconcile_all
task = asyncio.create_task(coordinator.reconcile_loop(interval=3600))
try:
await asyncio.wait_for(entered.wait(), timeout=1)
coordinator.wake_reconcile()
coordinator.wake_reconcile()
release.set()
assert await asyncio.wait_for(completed.get(), timeout=1) == 1
assert await asyncio.wait_for(completed.get(), timeout=1) == 2
await asyncio.sleep(0)
assert calls == 2
coordinator.wake_reconcile()
assert await asyncio.wait_for(completed.get(), timeout=1) == 3
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
asyncio.run(exercise())
def test_reconcile_loop_keeps_periodic_fallback_without_wakeups():
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
async def exercise():
completed = asyncio.Queue()
async def reconcile_all():
completed.put_nowait(True)
coordinator.reconcile_all = reconcile_all
task = asyncio.create_task(coordinator.reconcile_loop(interval=0.01))
try:
assert await asyncio.wait_for(completed.get(), timeout=1) is True
assert await asyncio.wait_for(completed.get(), timeout=1) is True
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
asyncio.run(exercise())
def payload(**overrides):
value = {
"schema_version": VPC_SCHEMA_VERSION,
"vpc_id": "vpc-hk-1",
"region": "hk",
"fabric_id": "fabric-hk",
"fabric_revision": 7,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-00000001",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
"routes": [],
"acls": [],
}
value.update(overrides)
return value
def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2,
operation_id="vpc-delete-00000001",
hosts=("host-a",), retiring_members=(), **overrides):
value = payload(
vpc_id=vpc_id,
desired_revision=desired_revision,
operation_id=operation_id,
subnets=[], members=[], routes=[], acls=[],
hosts=[{"host_id": host_id, "members": []} for host_id in hosts],
)
value["retiring_members"] = list(retiring_members)
value.update(overrides)
value["desired_digest"] = parse_vpc_delete_plan(value)[0].desired_digest
return value
def confirm_delete_hosts(plan, record):
"""Fill the exact bounded absence proof required for a delete receipt."""
for host in record.get("hosts", []):
host.update({
"state": "active",
"removed": True,
"transport_verified": True,
"private_dataplane_verified": True,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"member_ids": [],
"retiring_members": [],
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"proxy_ips": [],
"bridge": None,
"bridge_owner": None,
"nft_table": None,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"readback": "confirmed",
"rollback": "none",
})
def save_fabric_catalog(store, plan, *, plan_digest=None):
"""Persist the minimal digest-verified Fabric catalog shape."""
store.set("fabric_desired", plan.fabric_id, json.dumps({
"schema_version": 1,
"fabric_id": plan.fabric_id,
"region": plan.region,
"overlay_cidr": plan.overlay_cidr,
"revision": plan.revision,
"listen_port": plan.listen_port,
"plan_digest": plan.digest if plan_digest is None else plan_digest,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
}
for node in plan.nodes
],
}))
def retiring_member_a(*, host_id="host-a"):
return {
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": host_id,
"private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
}
def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state():
first = parse_vpc_plan(payload())
second = parse_vpc_plan(payload(
hosts=None,
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.10",
"subnet_id": "subnet-a",
}],
))
assert first.desired_digest == second.desired_digest
assert first.members[0].private_ip == "10.20.0.10"
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="192.0.2.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="10.20.0.0/24",
network_cidr="10.21.0.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(subnets=[
{"subnet_id": "a", "cidr": "10.20.0.0/25"},
{"subnet_id": "b", "cidr": "10.20.0.64/26"},
]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "public", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "0.0.0.0/0",
"destination_cidr": "10.20.0.10/32", "priority": 1,
}]))
def test_member_uuid_matches_existing_libvirt_identity_boundary():
# Existing panel/libvirt records only require the canonical UUID shape;
# operator-supplied UUIDs may use non-RFC version/variant nibbles. The
# engine must accept those persisted VM identities in a VPC DTO too.
legacy_uuid = "11111111-1111-6111-0111-111111111111"
parsed = parse_vpc_plan(payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": legacy_uuid,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
assert parsed.members[0].instance_uuid == legacy_uuid
def test_digest_matches_php_wire_shape_for_host_member_grouping():
wire = payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
)
# PHP computes desired_digest before attaching operation_id to the
# signed/replayed payload; operation_id is an envelope field, not part of
# desired document identity.
digest_wire = {key: value for key, value in wire.items()
if key != "operation_id"}
expected = __import__("hashlib").sha256(
json.dumps(digest_wire, sort_keys=True, separators=(",", ":"),
ensure_ascii=True).encode("utf-8")
).hexdigest()
parsed = parse_vpc_plan(wire)
assert parsed.desired_digest == expected
assert parsed.canonical()["hosts"] == [{
"host_id": "host-a", "members": ["member-a"]
}]
def test_plan_validates_routes_acl_ports_and_member_scope():
parsed = parse_vpc_plan(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32",
"next_hop_member_id": "member-a", "metric": 20}],
acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "port_from": 443,
"port_to": 443, "priority": 1}],
))
assert parsed.host_ids == {"host-a", "host-b"}
assert parsed.routes[0].next_hop_member_id == "member-a"
exact = parse_vpc_plan(payload(routes=[{
"route_id": "exact", "destination_cidr": "10.20.0.10/32",
"next_hop_member_id": "member-a",
}]))
assert exact.routes[0].destination_cidr == "10.20.0.10/32"
high_priority = parse_vpc_plan(payload(acls=[{
"acl_id": "high", "direction": "ingress", "action": "allow",
"protocol": "any", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "priority": 1_000_000,
}]))
assert high_priority.acls[0].priority == 1_000_000
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "bad", "destination_cidr": "0.0.0.0/0",
"next_hop_host_id": "host-a",
}]))
with pytest.raises(VpcError) as broad_route:
parse_vpc_plan(payload(routes=[{
"route_id": "broad", "destination_cidr": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
assert broad_route.value.error_code == "vpc_route_not_exact"
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "bad", "direction": "egress", "action": "allow",
"protocol": "icmp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.0/24", "port_from": 1,
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "alias-conflict",
"destination_cidr": "10.20.0.0/25",
"destination": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(hosts=[{
"host_id": "host-a", "members": ["member-a", "member-a"],
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(
hosts=[],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
def test_coordinator_persists_desired_children_and_blocks_without_fabric():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False))
result = asyncio.run(coordinator.apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["observed_revision"] == 0
assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False
assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"}
assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"}
assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"}
def test_guest_exec_ledger_is_immediate_bounded_private_and_restart_safe():
store = MemoryStore()
plan = parse_vpc_plan(payload())
member = plan.members[0]
coordinator = VpcCoordinator(store, FakeFabric())
state, persist, clear = coordinator._guest_exec_context(plan, member)
assert state == {}
core = {
"version": 1,
"phase": "pending",
"domain_uuid": member.instance_uuid,
"domain_runtime_id": 7,
"qemu_runtime_token": "a" * 32,
"guest_network_generation": "generation-1",
"probe_kind": "link",
"pid": 41,
}
persist(core)
key = f"{plan.vpc_id}.{member.instance_uuid}"
encoded = store.get(VPC_GUEST_EXEC_KIND, key)
assert encoded is not None
assert len(encoded.encode("utf-8")) <= 1024
assert json.loads(encoded)["pid"] == 41
restarted = VpcCoordinator(store, FakeFabric())
loaded, _persist_again, clear_again = restarted._guest_exec_context(
plan, member)
assert loaded == core
asyncio.run(coordinator.apply(payload()))
public = json.dumps(coordinator.get_status(plan.vpc_id), sort_keys=True)
assert '"pid"' not in public
assert "qemu_runtime_token" not in public
assert "a" * 32 not in public
clear_again()
assert store.get(VPC_GUEST_EXEC_KIND, key) is None
persist(core)
coordinator._purge_deleted_vpc(plan.vpc_id)
assert store.get(VPC_GUEST_EXEC_KIND, key) is None
def test_guest_exec_ledger_rejects_oversize_or_unknown_fields():
store = MemoryStore()
plan = parse_vpc_plan(payload())
member = plan.members[0]
key = f"{plan.vpc_id}.{member.instance_uuid}"
store.set(VPC_GUEST_EXEC_KIND, key, "x" * 1025)
with pytest.raises(VpcError, match="账本损坏"):
VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member)
bad = {
"vpc_id": plan.vpc_id,
"host_id": member.host_id,
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"version": 1,
"phase": "launch_intent",
"domain_uuid": member.instance_uuid,
"domain_runtime_id": 7,
"qemu_runtime_token": "a" * 32,
"guest_network_generation": "generation-1",
"probe_kind": "link",
"unexpected": True,
}
store.set(VPC_GUEST_EXEC_KIND, key, json.dumps(bad))
with pytest.raises(VpcError, match="账本损坏"):
VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member)
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert result["state"] == "applying"
assert result["next_stage_allowed"] is False
assert result["observed"]["fabric_gate"]["active"] is True
assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True
def test_recovery_reopens_only_the_control_plane_as_applying():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
asyncio.run(coordinator.apply(payload()))
coordinator.recover()
recovered = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert recovered["state"] == "applying"
assert recovered["next_stage_allowed"] is False
assert recovered["hosts"][0]["state"] == "applying"
assert recovered["hosts"][0]["transport_verified"] is True
def test_empty_vpc_cannot_open_fabric_gate():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
hosts=[], members=[],
)))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts"
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
assert result["state"] == "degraded"
assert result["fabric_gate"]["error_code"] == "vpc_no_members"
def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
)))
assert result["state"] == "applying"
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
digest = raw["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
assert observed["next_stage_allowed"] is True
assert observed["hosts"][0]["readback"] == {"safe": "ok"}
fabric.revision = 8
blocked = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert blocked["state"] == "degraded"
assert blocked["next_stage_allowed"] is False
def test_transport_only_observation_never_opens_private_dataplane_gate():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload()))
digest = result["desired_digest"]
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
}))
assert pending["state"] == "applying"
assert pending["next_stage_allowed"] is False
assert pending["private_dataplane_verified"] is False
assert pending["hosts"][0]["state"] == "applying"
assert pending["hosts"][0]["last_error_code"] == (
"vpc_private_dataplane_unverified"
)
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
)))
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert observed["state"] == "degraded"
assert observed["next_stage_allowed"] is False
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": False,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert inconsistent.value.error_code == "vpc_observation_invalid"
def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def wrong_region(fabric_id):
status = original(fabric_id)
status["region"] = "us"
return status
fabric.get_status = wrong_region
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert error.value.error_code == "vpc_fabric_region_mismatch"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_fabric_gate_rejects_underlay_endpoint_inside_tenant_cidr():
store = MemoryStore()
fabric = FakeFabric(
hosts=("host-a",), endpoint_ips=("10.20.0.50",),
)
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_vpc_rejects_endpoint_from_any_other_fabric_catalog_entry():
store = MemoryStore()
other = parse_fabric_plan({
"fabric_id": "fabric-sg",
"region": "sg",
"overlay_cidr": "100.64.2.0/24",
"revision": 3,
"nodes": [
{
"host_id": "host-c",
"overlay_ip": "100.64.2.1",
"endpoint": "10.20.0.50:51820",
},
{
"host_id": "host-d",
"overlay_ip": "100.64.2.2",
"endpoint": "203.0.113.4:51820",
},
],
})
store.set("fabric_desired", other.fabric_id, json.dumps({
"fabric_id": other.fabric_id,
"region": other.region,
"overlay_cidr": other.overlay_cidr,
"revision": other.revision,
"listen_port": other.listen_port,
"plan_digest": other.digest,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
}
for node in other.nodes
],
}))
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_vpc_fails_closed_on_corrupt_global_fabric_catalog_entry():
store = MemoryStore()
store.set("fabric_desired", "fabric-corrupt", "{}")
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert error.value.error_code == "vpc_fabric_record_corrupt"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_vpc_fails_closed_on_stale_global_fabric_plan_digest():
store = MemoryStore()
other = parse_fabric_plan({
"fabric_id": "fabric-sg",
"region": "sg",
"overlay_cidr": "100.64.2.0/24",
"revision": 3,
"nodes": [
{
"host_id": "host-c",
"overlay_ip": "100.64.2.1",
"endpoint": "203.0.113.3:51820",
},
{
"host_id": "host-d",
"overlay_ip": "100.64.2.2",
"endpoint": "203.0.113.4:51820",
},
],
})
store.set("fabric_desired", other.fabric_id, json.dumps({
"fabric_id": other.fabric_id,
"region": other.region,
"overlay_cidr": other.overlay_cidr,
"revision": other.revision,
"listen_port": other.listen_port,
"plan_digest": "0" * 64,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
}
for node in other.nodes
],
}))
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert error.value.error_code == "vpc_fabric_record_corrupt"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_vpc_apply_and_fabric_endpoint_change_share_catalog_admission_lock():
store = MemoryStore()
current = parse_fabric_plan({
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 7,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
store.set("fabric_desired", current.fabric_id, json.dumps({
"schema_version": 1,
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"plan_digest": current.digest,
"state": "active",
"next_stage_allowed": True,
"public_keys": {
"host-a": "unused-for-catalog-race-a",
"host-b": "unused-for-catalog-race-b",
},
"nodes": [
{
"host_id": "host-a",
"overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820",
"listen_port": 51820,
"state": "active",
"transport_verified": True,
},
{
"host_id": "host-b",
"overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820",
"listen_port": 51820,
"state": "active",
"transport_verified": True,
},
],
"updated_at": int(time.time()),
}))
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
coordinator = VpcCoordinator(
store, fabric, catalog_lock=fabric.catalog_mutation_lock)
entered = asyncio.Event()
release = asyncio.Event()
original_reconcile = coordinator._reconcile_dataplane
async def paused_reconcile(*args, **kwargs):
entered.set()
await release.wait()
return await original_reconcile(*args, **kwargs)
coordinator._reconcile_dataplane = paused_reconcile
candidate = {
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{
"host_id": "host-a",
"overlay_ip": "100.64.1.1",
"endpoint": "10.20.0.50:51820",
},
{
"host_id": "host-b",
"overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820",
},
],
}
async def run_race():
vpc_task = asyncio.create_task(coordinator.apply(payload()))
await entered.wait()
fabric_task = asyncio.create_task(fabric.apply(candidate))
await asyncio.sleep(0)
assert fabric_task.done() is False
release.set()
vpc_result = await vpc_task
with pytest.raises(FabricError) as error:
await fabric_task
return vpc_result, error.value
result, error = asyncio.run(run_race())
assert result["fabric_gate"]["active"] is True
assert error.error_code == "fabric_endpoint_vpc_conflict"
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert saved["cidr"] == "10.20.0.0/24"
def test_fabric_first_endpoint_change_blocks_conflicting_vpc_admission():
store = MemoryStore()
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
gate_fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(
store, gate_fabric,
catalog_lock=lock_owner.catalog_mutation_lock,
)
entered = asyncio.Event()
release = asyncio.Event()
async def publish_endpoint(_plan):
entered.set()
await release.wait()
gate_fabric.endpoint_ips = ("10.20.0.50",)
return {"state": "active"}
lock_owner._apply_locked = publish_endpoint
candidate = {
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{
"host_id": "host-a",
"overlay_ip": "100.64.1.1",
"endpoint": "10.20.0.50:51820",
},
{
"host_id": "host-b",
"overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820",
},
],
}
async def run_race():
fabric_task = asyncio.create_task(lock_owner.apply(candidate))
await entered.wait()
vpc_task = asyncio.create_task(coordinator.apply(payload()))
await asyncio.sleep(0)
assert vpc_task.done() is False
release.set()
await fabric_task
with pytest.raises(VpcError) as error:
await vpc_task
return error.value
error = asyncio.run(run_race())
assert error.error_code == "vpc_fabric_endpoint_overlap"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_cancelled_fabric_apply_drains_thread_before_releasing_catalog_lock():
store = MemoryStore()
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
coordinator = VpcCoordinator(
store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock)
entered = threading.Event()
release = threading.Event()
async def blocked_apply(_plan):
def mutate():
entered.set()
assert release.wait(timeout=2)
store.set("test", "fabric-thread", "settled")
await asyncio.to_thread(mutate)
return {"state": "settled"}
fabric._apply_locked = blocked_apply
candidate = {
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{
"host_id": "host-a",
"overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820",
},
{
"host_id": "host-b",
"overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820",
},
],
}
async def run_cancel():
fabric_task = asyncio.create_task(fabric.apply(candidate))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
vpc_task = asyncio.create_task(coordinator.apply(payload()))
await asyncio.sleep(0)
assert vpc_task.done() is False
fabric_task.cancel()
await asyncio.sleep(0)
assert fabric_task.done() is False
assert vpc_task.done() is False
assert store.get("test", "fabric-thread") is None
release.set()
with pytest.raises(asyncio.CancelledError):
await fabric_task
return await asyncio.wait_for(vpc_task, timeout=1)
result = asyncio.run(run_cancel())
assert store.get("test", "fabric-thread") == "settled"
assert result["vpc_id"] == "vpc-hk-1"
assert store.get("vpc_desired", "vpc-hk-1") is not None
def test_cancelled_vpc_apply_drains_thread_and_saves_before_fabric_enters():
store = MemoryStore()
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
coordinator = VpcCoordinator(
store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock)
vpc_thread_entered = threading.Event()
vpc_thread_release = threading.Event()
fabric_entered = asyncio.Event()
async def blocked_reconcile(_plan, record, _gate, _previous_plan):
def mutate():
vpc_thread_entered.set()
assert vpc_thread_release.wait(timeout=2)
record["thread_mutation_settled"] = True
await asyncio.to_thread(mutate)
async def observed_fabric_apply(_plan):
fabric_entered.set()
return {"state": "settled"}
coordinator._reconcile_dataplane = blocked_reconcile
fabric._apply_locked = observed_fabric_apply
candidate = {
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{
"host_id": "host-a",
"overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820",
},
{
"host_id": "host-b",
"overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820",
},
],
}
async def run_cancel():
vpc_task = asyncio.create_task(coordinator.apply(payload()))
for _attempt in range(100):
if vpc_thread_entered.is_set():
break
await asyncio.sleep(0.01)
assert vpc_thread_entered.is_set()
fabric_task = asyncio.create_task(fabric.apply(candidate))
await asyncio.sleep(0)
assert fabric_entered.is_set() is False
vpc_task.cancel()
await asyncio.sleep(0)
assert vpc_task.done() is False
assert fabric_entered.is_set() is False
assert store.get("vpc_desired", "vpc-hk-1") is None
vpc_thread_release.set()
with pytest.raises(asyncio.CancelledError):
await vpc_task
await asyncio.wait_for(fabric_task, timeout=1)
asyncio.run(run_cancel())
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert saved["thread_mutation_settled"] is True
def test_cancelled_root_delete_drains_host_mutation_before_catalog_unlock():
store = MemoryStore()
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)),
catalog_lock=fabric.catalog_mutation_lock,
)
delete_thread_entered = threading.Event()
delete_thread_release = threading.Event()
fabric_entered = asyncio.Event()
async def blocked_delete(plan, record, _gate, _previous_plan, **_kwargs):
def mutate():
delete_thread_entered.set()
assert delete_thread_release.wait(timeout=2)
confirm_delete_hosts(plan, record)
await asyncio.to_thread(mutate)
async def observed_fabric_apply(_plan):
fabric_entered.set()
return {"state": "settled"}
coordinator._reconcile_dataplane = blocked_delete
fabric._apply_locked = observed_fabric_apply
target = delete_payload(
vpc_id="vpc-never-applied",
desired_revision=3,
operation_id="vpc-delete-cancel-drain-0001",
cidr="10.21.0.0/24",
)
candidate = {
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
}
async def run_cancel():
delete_task = asyncio.create_task(coordinator.delete(
"vpc-never-applied", target))
for _attempt in range(100):
if delete_thread_entered.is_set():
break
await asyncio.sleep(0.01)
assert delete_thread_entered.is_set()
# The host-bearing absent-target delete has already published its
# temporary CIDR reservation, but still owns the shared catalog lock.
assert store.get("vpc_desired", "vpc-never-applied") is not None
fabric_task = asyncio.create_task(fabric.apply(candidate))
await asyncio.sleep(0)
assert fabric_entered.is_set() is False
delete_task.cancel()
await asyncio.sleep(0)
assert delete_task.done() is False
assert fabric_entered.is_set() is False
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None
delete_thread_release.set()
with pytest.raises(asyncio.CancelledError):
await delete_task
await asyncio.wait_for(fabric_task, timeout=1)
asyncio.run(run_cancel())
receipt = json.loads(store.get(
VPC_DELETE_RECEIPT_KIND, "vpc-never-applied"))
assert receipt["state"] == "deleted"
assert receipt["readback"] == "confirmed"
assert store.get("vpc_desired", "vpc-never-applied") is None
assert fabric_entered.is_set() is True
@pytest.mark.parametrize("entrypoint", ["observe", "reconcile", "instance"])
def test_fabric_gate_change_precedes_every_vpc_evidence_publication(
entrypoint):
store = MemoryStore()
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
gate_fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(
store, gate_fabric,
catalog_lock=lock_owner.catalog_mutation_lock,
)
initial = asyncio.run(coordinator.apply(payload()))
digest = initial["desired_digest"]
active_observation = {
"host-a": {
"state": "active",
"observed_revision": 1,
"observed_digest": digest,
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"config_observed_revision": 1,
"config_observed_digest": digest,
},
}
active = asyncio.run(coordinator.observe(
"vpc-hk-1", active_observation))
assert active["next_stage_allowed"] is True
async def run_race():
fabric_entered = asyncio.Event()
release_fabric = asyncio.Event()
async def change_fabric_gate():
async with lock_owner.catalog_mutation_lock():
fabric_entered.set()
await release_fabric.wait()
gate_fabric.revision = 8
gate_fabric.state = "degraded"
gate_fabric.gate = False
fabric_task = asyncio.create_task(change_fabric_gate())
await fabric_entered.wait()
if entrypoint == "observe":
vpc_task = asyncio.create_task(coordinator.observe(
"vpc-hk-1", active_observation))
elif entrypoint == "reconcile":
vpc_task = asyncio.create_task(coordinator.reconcile("vpc-hk-1"))
else:
vpc_task = asyncio.create_task(coordinator.reconcile_instance(
INSTANCE_A,
host_id="host-a",
expected={
"required": True,
"vpc_id": "vpc-hk-1",
"member_id": "member-a",
},
))
await asyncio.sleep(0)
assert vpc_task.done() is False
release_fabric.set()
await fabric_task
return await asyncio.wait_for(vpc_task, timeout=1)
result = asyncio.run(run_race())
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["fabric_gate"]["fabric_revision"] == 8
assert durable["fabric_gate"]["active"] is False
assert durable["next_stage_allowed"] is False
assert durable["state"] != "active"
if entrypoint == "instance":
assert result["readback"] != "confirmed"
else:
assert result["next_stage_allowed"] is False
def test_observe_locks_active_and_retiring_vm_identities_in_sorted_order():
store = MemoryStore()
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
asyncio.run(coordinator.apply(payload(hosts=[{
"host_id": "host-a",
"members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a"},
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a"},
],
}])))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2,
operation_id="vpc-observe-lock-order-0002",
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring_members"][0][
"instance_uuid"] == INSTANCE_B
acquired.clear()
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {
"state": "active",
"observed_revision": 2,
"observed_digest": second["desired_digest"],
"transport_verified": True,
"private_dataplane_verified": True,
},
}))
assert acquired == [
("host-a", INSTANCE_A),
("host-a", INSTANCE_B),
]
@pytest.mark.parametrize("entrypoint", ["reconcile", "instance"])
def test_cancelled_vpc_retry_drains_host_thread_before_catalog_unlock(
entrypoint):
store = MemoryStore()
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)),
catalog_lock=lock_owner.catalog_mutation_lock,
)
asyncio.run(coordinator.apply(payload()))
thread_entered = threading.Event()
thread_release = threading.Event()
async def blocked_reconcile(_plan, record, _gate, _previous_plan,
**_kwargs):
def mutate():
thread_entered.set()
assert thread_release.wait(timeout=2)
record["hosts"][0]["cancel_drain_marker"] = entrypoint
await asyncio.to_thread(mutate)
coordinator._reconcile_dataplane = blocked_reconcile
async def run_cancel():
fabric_entered = asyncio.Event()
async def observed_fabric_apply(_plan):
fabric_entered.set()
return {"state": "settled"}
lock_owner._apply_locked = observed_fabric_apply
if entrypoint == "reconcile":
vpc_task = asyncio.create_task(
coordinator.reconcile("vpc-hk-1"))
else:
vpc_task = asyncio.create_task(coordinator.reconcile_instance(
INSTANCE_A,
host_id="host-a",
expected={
"required": True,
"vpc_id": "vpc-hk-1",
"member_id": "member-a",
},
))
for _attempt in range(100):
if thread_entered.is_set():
break
await asyncio.sleep(0.01)
assert thread_entered.is_set()
fabric_task = asyncio.create_task(lock_owner.apply({
"fabric_id": "fabric-hk",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 8,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
}))
await asyncio.sleep(0)
assert fabric_entered.is_set() is False
vpc_task.cancel()
await asyncio.sleep(0)
assert vpc_task.done() is False
assert fabric_entered.is_set() is False
thread_release.set()
with pytest.raises(asyncio.CancelledError):
await vpc_task
await asyncio.wait_for(fabric_task, timeout=1)
assert fabric_entered.is_set() is True
asyncio.run(run_cancel())
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["cancel_drain_marker"] == entrypoint
def test_fabric_gate_rejects_tampered_identity_without_saving():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def tampered(fabric_id):
status = original(fabric_id)
status["fabric_id"] = "other-fabric"
return status
fabric.get_status = tampered
with pytest.raises(VpcError) as error:
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert error.value.error_code == "vpc_fabric_identity_mismatch"
assert store.get("vpc_desired", "vpc-hk-1") is None
def test_fabric_gate_keeps_revision_mismatch_as_reconcilable_state():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def tampered(fabric_id):
status = original(fabric_id)
status["revision"] = True
return status
fabric.get_status = tampered
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["next_stage_allowed"] is False
assert result["state"] == "degraded"
def test_same_vpc_revision_and_digest_cannot_borrow_another_operation_receipt(monkeypatch):
async def run():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
first = await coordinator.apply(payload(operation_id="a" * 32))
before = store.get("vpc_desired", "vpc-hk-1")
retries = []
async def observe_only(*_args, **_kwargs):
retries.append(True)
monkeypatch.setattr(coordinator, "_reconcile_pending_hosts", observe_only)
with pytest.raises(VpcError) as conflict:
await coordinator.apply(payload(operation_id="b" * 32))
assert conflict.value.status == 409
assert conflict.value.error_code == "vpc_revision_conflict"
assert store.get("vpc_desired", "vpc-hk-1") == before
assert retries == []
replay = await coordinator.apply(payload(operation_id="a" * 32))
assert replay["operation_id"] == first["operation_id"]
assert retries == [True]
asyncio.run(run())
def test_coordinator_revision_conflict_and_tampered_record_fail_closed():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
first = asyncio.run(coordinator.apply(payload()))
replay = asyncio.run(coordinator.apply(payload()))
assert replay["desired_digest"] == first["desired_digest"]
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.apply(payload(
operation_id="vpc-op-00000002",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert error.value.error_code == "vpc_revision_conflict"
with pytest.raises(VpcError) as invalid_revision:
asyncio.run(coordinator.apply(payload(
desired_revision=0, operation_id="vpc-op-00000003",
)))
assert invalid_revision.value.error_code == "vpc_invalid"
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)))
with pytest.raises(VpcError) as stale:
asyncio.run(coordinator.apply(payload(
desired_revision=1, operation_id="vpc-op-00000005",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert stale.value.error_code == "vpc_revision_conflict"
tampered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
tampered["desired_digest"] = "0" * 64
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as corrupt:
asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert corrupt.value.error_code == "vpc_record_corrupt"
tampered["vpc_id"] = "other-vpc"
tampered["desired_digest"] = first["desired_digest"]
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as mismatched:
coordinator.get_status("vpc-hk-1")
assert mismatched.value.error_code == "vpc_record_corrupt"
# Restore a structurally valid root and corrupt only observed evidence;
# a newer target must not overwrite that evidence silently.
restored = json.loads(store.get("vpc_desired", "vpc-hk-1"))
restored["vpc_id"] = "vpc-hk-1"
restored["desired_revision"] = 2
restored["desired_digest"] = parse_vpc_plan(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)).desired_digest
restored["observed_revision"] = 99
store.set("vpc_desired", "vpc-hk-1", json.dumps(restored))
with pytest.raises(VpcError) as observed_corrupt:
asyncio.run(coordinator.apply(payload(
desired_revision=3, operation_id="vpc-op-00000006",
)))
assert observed_corrupt.value.error_code == "vpc_record_corrupt"
def test_recover_invalidates_observed_gate_and_record_round_trips():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
raw["state"] = "active"
raw["next_stage_allowed"] = True
raw["observed_revision"] = 1
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
transport_verified=True,
private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert recovered["state"] == "uncertain"
assert recovered["next_stage_allowed"] is False
assert recovered["observed_revision"] == 1
assert recovered["observed_digest"] == recovered["desired_digest"]
assert recovered["fabric_gate"]["active"] is False
assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending"
assert recovered["hosts"][0]["state"] == "uncertain"
assert recovered["hosts"][0]["transport_verified"] is False
assert recovered["hosts"][0]["private_dataplane_verified"] is False
assert recovered["hosts"][0]["observed_revision"] == 1
assert _plan_from_record(recovered).vpc_id == "vpc-hk-1"
def test_removed_children_are_tombstoned_until_observed_cleanup():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
changed = payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}],
members=[],
)
asyncio.run(coordinator.apply(changed))
old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a"))
assert old["state"] == "removed"
assert old["removed"] is True
def test_new_revision_drops_host_with_exact_prior_removal_proof():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
first_payload = payload(
hosts=[{"host_id": "host-a", "members": []}], members=[])
asyncio.run(coordinator.apply(first_payload))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
confirm_delete_hosts(parse_vpc_plan(first_payload), durable)
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
probed = []
async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs):
probed.extend(host["host_id"] for host in record["hosts"])
coordinator._reconcile_dataplane = capture_hosts
result = asyncio.run(coordinator.apply(payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)))
assert probed == ["host-b"]
assert [host["host_id"] for host in result["hosts"]] == ["host-b"]
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert [host["host_id"] for host in saved["hosts"]] == ["host-b"]
def test_new_revision_retires_host_without_exact_prior_removal_proof():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
first_payload = payload()
asyncio.run(coordinator.apply(first_payload))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
# A marker without the exact typed absence/readback fields is not proof.
durable["hosts"][0]["removed"] = True
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
probed = []
async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs):
probed.extend(host["host_id"] for host in record["hosts"])
coordinator._reconcile_dataplane = capture_hosts
result = asyncio.run(coordinator.apply(payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)))
assert probed == ["host-b", "host-a"]
second_durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
retiring = next(host for host in second_durable["hosts"]
if host["host_id"] == "host-a")
assert retiring["retiring"] is True
assert retiring.get("removed") is not True
assert retiring["member_ids"] == ["member-a"]
assert retiring["retiring_members"] == [{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"host_id": "host-a",
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
}]
probed.clear()
third_payload = payload(
desired_revision=3,
operation_id="vpc-op-00000003",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)
third = asyncio.run(coordinator.apply(third_payload))
assert probed == ["host-b", "host-a"]
third_durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
carried = next(host for host in third_durable["hosts"]
if host["host_id"] == "host-a")
assert carried["retiring"] is True
assert carried.get("removed") is not True
assert carried["member_ids"] == retiring["member_ids"]
assert carried["retiring_members"] == retiring["retiring_members"]
child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a"))
assert child["state"] == "deleting"
assert child.get("removed") is not True
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
host_a = next(host for host in durable["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(parse_vpc_plan(third_payload), {"hosts": [host_a]})
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
coordinator._save_children(durable)
probed.clear()
fourth = asyncio.run(coordinator.apply(payload(
desired_revision=4,
operation_id="vpc-op-00000004",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)))
assert probed == ["host-b"]
assert [host["host_id"] for host in fourth["hosts"]] == ["host-b"]
child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a"))
assert coordinator._terminal_removal(child)
assert child["desired_revision"] == 3
assert child["observed_revision"] == 3
root_delete_calls = []
async def confirm_root_delete(
plan, record, gate, previous_plan=None, **kwargs):
root_delete_calls.append(
[host["host_id"] for host in record["hosts"]])
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm_root_delete
member_b = {
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"host_id": "host-b", "private_ip_cidr": "10.20.0.11/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
}
deleted = asyncio.run(coordinator.delete("vpc-hk-1", delete_payload(
desired_revision=5,
operation_id="vpc-delete-historical-proof-0001",
hosts=("host-a", "host-b"),
retiring_members=(retiring_member_a(), member_b),
)))
assert root_delete_calls == [["host-b"]]
assert deleted["removed_host_ids"] == ["host-a", "host-b"]
@pytest.mark.parametrize("corruption", ["not-list", "non-object", "missing"])
def test_new_revision_rejects_corrupt_durable_retiring_member_closure(corruption):
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
asyncio.run(coordinator.apply(payload()))
async def leave_pending(plan, record, gate, previous_plan=None, **kwargs):
return None
coordinator._reconcile_dataplane = leave_pending
asyncio.run(coordinator.apply(payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
host_a = next(host for host in durable["hosts"]
if host["host_id"] == "host-a")
if corruption == "not-list":
host_a["retiring_members"] = {"member_id": "member-a"}
elif corruption == "non-object":
host_a["retiring_members"] = [None]
else:
host_a["retiring_members"] = []
tampered = json.dumps(durable)
store.set("vpc_desired", "vpc-hk-1", tampered)
with pytest.raises(VpcError) as rejected:
asyncio.run(coordinator.apply(payload(
desired_revision=3,
operation_id="vpc-op-00000003",
hosts=[{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]}],
)))
assert rejected.value.error_code == "vpc_record_corrupt"
assert store.get("vpc_desired", "vpc-hk-1") == tampered
def test_apply_replay_retries_only_hosts_without_exact_removal_proof_after_restart():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
target = payload(hosts=[
{"host_id": "host-a", "members": []},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]},
])
calls = []
async def remove_a_then_recover_b(
plan, record, gate, previous_plan=None, **kwargs):
host_ids = [host["host_id"] for host in record["hosts"]]
calls.append(host_ids)
if len(calls) == 1:
assert host_ids == ["host-a", "host-b"]
host_a = next(host for host in record["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(plan, {"hosts": [host_a]})
host_b = next(host for host in record["hosts"]
if host["host_id"] == "host-b")
host_b.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_host_unavailable",
})
return
# Host A is now offline. Reaching this branch with A in the execution
# closure would model the liveness regression this test prevents.
assert host_ids == ["host-b"]
record["hosts"][0].update({
"state": "active",
"removed": False,
"transport_verified": True,
"private_dataplane_verified": True,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"last_error_code": None,
})
coordinator._reconcile_dataplane = remove_a_then_recover_b
first = asyncio.run(coordinator.apply(target))
assert first["state"] != "active"
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
durable_hosts = {host["host_id"]: host for host in durable["hosts"]}
assert coordinator._confirmed_removal(
durable_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
assert durable_hosts["host-b"]["state"] == "uncertain"
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]}
assert coordinator._confirmed_removal(
recovered_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
assert recovered_hosts["host-b"]["state"] == "uncertain"
result = asyncio.run(coordinator.apply(target))
assert calls == [["host-a", "host-b"], ["host-b"]]
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
final_hosts = {host["host_id"]: host for host in result["hosts"]}
assert coordinator._confirmed_removal(
final_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
assert final_hosts["host-b"]["state"] == "active"
def test_reconcile_after_restart_retries_only_pending_normal_apply_hosts():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
target = payload(hosts=[
{"host_id": "host-a", "members": []},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]},
])
async def partial(plan, record, gate, previous_plan=None, **kwargs):
host_a = next(host for host in record["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(plan, {"hosts": [host_a]})
host_b = next(host for host in record["hosts"]
if host["host_id"] == "host-b")
host_b.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_host_unavailable",
})
coordinator._reconcile_dataplane = partial
asyncio.run(coordinator.apply(target))
coordinator.recover()
restarted = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
calls = []
async def recover_b(plan, record, gate, previous_plan=None, **kwargs):
host_ids = [host["host_id"] for host in record["hosts"]]
calls.append(host_ids)
# Host A is intentionally unreachable after restart. It must not be
# part of the execution closure once its exact absence proof is
# durable; only the previously unavailable host B may be contacted.
assert host_ids == ["host-b"]
record["hosts"][0].update({
"state": "active",
"removed": False,
"transport_verified": True,
"private_dataplane_verified": True,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"last_error_code": None,
})
restarted._reconcile_dataplane_unlocked = recover_b
result = asyncio.run(restarted.reconcile("vpc-hk-1"))
assert calls == [["host-b"]]
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
final_hosts = {host["host_id"]: host for host in result["hosts"]}
plan = parse_vpc_plan(target)
assert restarted._confirmed_removal(
final_hosts["host-a"], plan.desired_revision, plan.desired_digest)
assert final_hosts["host-b"]["state"] == "active"
def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
first = asyncio.run(coordinator.apply(payload()))
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
)))
assert second["state"] == "applying"
assert second["next_stage_allowed"] is False
assert second["observed_revision"] == 1
assert second["observed_digest"] == digest
assert second["hosts"][0]["observed_revision"] == 1
def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None, **_kwargs):
host = record["hosts"][0]
requested = (private_link_states or {}).get(INSTANCE_A, "preserve")
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
config_verified=True,
config_observed_revision=plan.desired_revision,
config_observed_digest=plan.desired_digest,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
"xml_verified": True,
"config_verified": True,
"seed_verified": True,
"restart_required": False,
"domain_running": True,
"guest_network_verified": requested != "down",
"guest_network_readback": (
"confirmed" if requested != "down" else "mismatch"),
"guest_reason_code": (
None if requested != "down"
else "vpc_private_guest_link_mismatch"),
"guest_private_mac": (derive_private_mac(
"vpc-hk-1", "member-a")
if requested != "down" else None),
"guest_private_ip_cidr": (
"10.20.0.10/32" if requested != "down" else None),
"guest_link_state": (
"up" if requested != "down" else None),
"guest_routes": [],
"link_state": "down" if requested == "down" else "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="down",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["readback"] == "confirmed"
assert state["state"] == "suspended"
assert state["vpc_id"] == "vpc-hk-1"
assert state["member_id"] == "member-a"
assert state["link_state"] == "down"
assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a")
# Preserve does not implicitly request link-up from the coordinator.
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="preserve",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["link_state"] == "up"
def test_instance_reconcile_invalidates_stale_proof_when_fabric_is_inactive():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def confirm(plan, record, gate, previous_plan=None,
private_link_states=None, **_kwargs):
record["hosts"][0].update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True, config_verified=True,
config_observed_revision=plan.desired_revision,
config_observed_digest=plan.desired_digest,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0",
"private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac(
"vpc-hk-1", "member-a"),
"xml_verified": True, "config_verified": True,
"seed_verified": True, "restart_required": False,
"domain_running": True, "guest_network_verified": True,
"guest_network_readback": "confirmed",
"guest_reason_code": None,
"guest_private_mac": derive_private_mac(
"vpc-hk-1", "member-a"),
"guest_private_ip_cidr": "10.20.0.10/32",
"guest_link_state": "up",
"guest_routes": [], "link_state": "up",
}],
)
coordinator._reconcile_dataplane = confirm
active = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert active["readback"] == "confirmed"
coordinator.fabric = FakeFabric(
state="degraded", gate=False, hosts=("host-a",))
coordinator._reconcile_dataplane = (
VpcCoordinator._reconcile_dataplane.__get__(
coordinator, VpcCoordinator))
stale = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert stale["state"] == "uncertain"
assert stale["readback"] == "missing"
def test_removed_member_reconcile_uses_durable_retiring_tombstone():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None, **_kwargs):
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_state": "removed"},
))
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
assert state["link_state"] == "absent"
def test_instance_reconcile_rejects_stale_revision_before_host_apply():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
calls = []
async def fake_reconcile(*args, **kwargs):
calls.append((args, kwargs))
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={
"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_revision": 99,
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
},
))
assert calls == []
assert state["readback"] == "missing"
assert state["reason_code"] == "vpc_private_expectation_mismatch"
def test_instance_reconcile_rejects_host_mac_drift():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None, **_kwargs):
record["hosts"][0].update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
config_verified=True,
config_observed_revision=plan.desired_revision,
config_observed_digest=plan.desired_digest,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": "02:00:00:00:00:01", "xml_verified": True,
"config_verified": True, "seed_verified": True,
"restart_required": False,
"domain_running": True,
"guest_network_verified": True,
"guest_network_readback": "confirmed",
"guest_reason_code": None,
"guest_private_mac": "02:00:00:00:00:01",
"guest_private_ip_cidr": "10.20.0.10/32",
"guest_link_state": "up",
"guest_routes": [],
"link_state": "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert state["readback"] == "missing"
assert state["state"] == "uncertain"
def test_removed_member_reconcile_locks_retiring_vm_identity():
store = MemoryStore()
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
# One host pass removes every retiring member. Keep a sibling tombstone
# in the durable row to prove the lock set is not limited to the request
# target.
durable["hosts"][0]["retiring_members"].append({
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"host_id": "host-a", "private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
})
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
acquired.clear()
async def fake_unlocked(plan, record, gate, previous_plan=None,
private_link_states=None, **_kwargs):
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane_unlocked = fake_unlocked
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a", "desired_state": "removed",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert ("host-a", INSTANCE_A) in acquired
assert ("host-a", INSTANCE_B) in acquired
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
key = f"host-scope.{INSTANCE_A}"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "staged",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
# A same-operation queued/failed row has not established a terminal
# rescue state and must remain a barrier.
for status in ("queued", "running", "exiting", "failed"):
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": status,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
def test_lifecycle_conflicts_follow_pending_host_execution_closure():
host_a_store = MemoryStore()
host_b_store = MemoryStore()
host_a_store.set("reinstall_job", "job-a", json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "reinstall-op-a",
"status": "running",
}))
registry = SimpleNamespace(components={
"host-a": SimpleNamespace(store=host_a_store),
"host-b": SimpleNamespace(store=host_b_store),
})
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=registry,
)
plan = parse_vpc_plan(payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]},
]))
gate = {"active": True, "available": True}
full_record = coordinator._desired_record(
plan, gate, state="applying")
host_b = next(
host for host in full_record["hosts"]
if host["host_id"] == "host-b")
execution_record = dict(full_record)
execution_record["hosts"] = [dict(host_b)]
# Host A is not part of this pending-host retry, so its lifecycle job
# cannot starve host B's otherwise independent convergence pass.
assert coordinator._mark_lifecycle_conflicts(
plan, plan, execution_record) is False
assert execution_record["hosts"][0].get("last_error_code") is None
# A full mutation still spans both hosts and keeps the original barrier.
assert coordinator._mark_lifecycle_conflicts(
plan, plan, full_record) is True
hosts = {host["host_id"]: host for host in full_record["hosts"]}
assert hosts["host-a"]["last_error_code"] == (
"vpc_vm_lifecycle_conflict")
assert hosts["host-b"].get("last_error_code") is None
def test_lifecycle_conflicts_keep_same_host_sibling_barrier():
host_store = MemoryStore()
host_store.set("snapshot_job", "job-a", json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "snapshot-op-a",
"status": "running",
}))
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-b",)),
registry=SimpleNamespace(components={
"host-b": SimpleNamespace(store=host_store),
}),
)
plan = parse_vpc_plan(payload(hosts=[{
"host_id": "host-b", "members": [
{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
},
{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
},
],
}]))
record = coordinator._desired_record(
plan, {"active": True, "available": True}, state="applying")
# A selected host pass may update shared bridge/WG/XML state for every
# local member. A sibling lifecycle job on that same host remains a hard
# barrier even when the originally pending member is different.
assert coordinator._mark_lifecycle_conflicts(
plan, plan, record) is True
assert record["hosts"][0]["last_error_code"] == (
"vpc_vm_lifecycle_conflict")
def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict"
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "reinstall_committed": True,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "phase": "result_uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
# Rescue job uncertainty is never exempted by the operation id alone;
# only the vm_rescue active/staged row proves its own terminal XML state.
store.delete("reinstall_job", "job-1")
store.set("rescue_job", "job-2", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
@pytest.mark.parametrize("identity_change", [
{"region": "sg"},
{"fabric_id": "fabric-sg"},
{"fabric_revision": 8},
{"cidr": "10.21.0.0/24"},
])
def test_root_delete_rejects_catalog_identity_change_without_mutation(
identity_change):
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
asyncio.run(coordinator.apply(payload()))
before = dict(store.values)
async def must_not_mutate(*_args, **_kwargs):
raise AssertionError("identity mismatch reached host mutation")
coordinator._reconcile_dataplane = must_not_mutate
fabric.get_status = lambda _fabric_id: (_ for _ in ()).throw(
AssertionError("identity mismatch reached Fabric gate"))
target = delete_payload(
desired_revision=2,
operation_id="vpc-delete-identity-mismatch-0001",
**identity_change,
)
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert error.value.status == 409
assert error.value.error_code == "vpc_delete_identity_mismatch"
# Compare the store's exact encoded values: a rejected delete must not
# rewrite timestamps, child projections, or the previous desired bytes.
assert store.values == before
def test_root_delete_without_previous_rejects_global_fabric_endpoint_overlap():
store = MemoryStore()
conflicting = parse_fabric_plan({
"fabric_id": "fabric-sg",
"region": "sg",
"overlay_cidr": "100.64.2.0/24",
"revision": 3,
"nodes": [
{"host_id": "host-c", "overlay_ip": "100.64.2.1",
"endpoint": "10.21.0.50:51820"},
{"host_id": "host-d", "overlay_ip": "100.64.2.2",
"endpoint": "203.0.113.4:51820"},
],
})
save_fabric_catalog(store, conflicting)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
async def must_not_mutate(*_args, **_kwargs):
raise AssertionError("conflicting delete reached host mutation")
coordinator._reconcile_dataplane = must_not_mutate
before = dict(store.values)
target = delete_payload(
vpc_id="vpc-never-applied",
desired_revision=3,
operation_id="vpc-delete-catalog-overlap-0001",
cidr="10.21.0.0/24",
)
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.delete("vpc-never-applied", target))
assert error.value.status == 409
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
assert store.values == before
def test_root_delete_without_previous_fails_closed_on_stale_fabric_digest():
store = MemoryStore()
fabric_plan = parse_fabric_plan({
"fabric_id": "fabric-sg",
"region": "sg",
"overlay_cidr": "100.64.2.0/24",
"revision": 3,
"nodes": [
{"host_id": "host-c", "overlay_ip": "100.64.2.1",
"endpoint": "203.0.113.3:51820"},
{"host_id": "host-d", "overlay_ip": "100.64.2.2",
"endpoint": "203.0.113.4:51820"},
],
})
save_fabric_catalog(store, fabric_plan, plan_digest="0" * 64)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
before = dict(store.values)
target = delete_payload(
vpc_id="vpc-never-applied",
desired_revision=3,
operation_id="vpc-delete-stale-fabric-0001",
cidr="10.21.0.0/24",
)
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.delete("vpc-never-applied", target))
assert error.value.status == 500
assert error.value.error_code == "vpc_fabric_record_corrupt"
assert store.values == before
def test_root_delete_accepts_revision_jump_and_superseded_unknown_host():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
asyncio.run(coordinator.apply(payload()))
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append({
"host_ids": sorted(host["host_id"] for host in record["hosts"]),
"retiring": all(host.get("retiring") is True
for host in record["hosts"]),
"operation_id": kwargs.get("operation_id"),
})
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
desired_revision=4,
operation_id="vpc-delete-revision-jump-0001",
hosts=("host-a", "host-b"),
retiring_members=(retiring_member_a(),),
)
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result == {
"vpc_id": "vpc-hk-1",
"operation_id": "vpc-delete-revision-jump-0001",
"evidence_version": 2,
"desired_revision": 4,
"desired_digest": target["desired_digest"],
"state": "deleted",
"deleted": True,
"readback": "confirmed",
"removed_host_ids": ["host-a", "host-b"],
"completed_at": result["completed_at"],
}
assert result["completed_at"] > 0
assert calls == [{
"host_ids": ["host-a", "host-b"],
"retiring": True,
"operation_id": "vpc-delete-revision-jump-0001",
}]
assert store.get("vpc_desired", "vpc-hk-1") is None
assert store.list("vpc_host") == {}
durable_receipt = json.loads(store.get(
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
assert durable_receipt["readback"] == "confirmed"
assert durable_receipt["completed_at"] == result["completed_at"]
assert "retiring_members" not in durable_receipt
# A receipt written by an earlier engine build is compacted on replay;
# the digest retains the manifest commitment without permanently keeping
# member UUID/private IP/MAC identifiers after successful deletion.
legacy_receipt = dict(durable_receipt)
legacy_receipt["retiring_members"] = target["retiring_members"]
store.set(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1", json.dumps(legacy_receipt))
replay = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert replay == result
compacted_raw = store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")
compacted = json.loads(compacted_raw)
assert "retiring_members" not in compacted
assert INSTANCE_A not in compacted_raw
assert "10.20.0.10" not in compacted_raw
assert retiring_member_a()["private_mac"] not in compacted_raw
conflict = dict(target, operation_id="vpc-delete-revision-jump-0002")
with pytest.raises(VpcError) as changed:
asyncio.run(coordinator.delete("vpc-hk-1", conflict))
assert changed.value.status == 409
assert changed.value.error_code == "vpc_delete_receipt_conflict"
with pytest.raises(VpcError) as recreated:
asyncio.run(coordinator.apply(payload(
desired_revision=5, operation_id="vpc-after-delete-0001")))
assert recreated.value.error_code == "vpc_deleted"
def test_delete_manifest_accepts_unicast_with_odd_last_octet_only():
derived = derive_private_mac("never", "m")
assert derived.endswith(":49")
valid = delete_payload(
vpc_id="never", cidr="10.24.0.0/24", desired_revision=1,
operation_id="vpc-delete-mac-boundary-0001", hosts=("host-a",),
retiring_members=({
"member_id": "m", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.24.0.10/32",
"private_mac": derived,
},),
)
assert parse_vpc_delete_plan(valid)[1][0].private_mac == derived
invalid = dict(valid)
invalid.pop("desired_digest")
invalid["retiring_members"] = [{
**valid["retiring_members"][0],
"private_mac": "13:00:00:00:00:48",
}]
with pytest.raises(VpcError) as multicast:
parse_vpc_delete_plan(invalid)
assert multicast.value.error_code == "vpc_delete_manifest_invalid"
invalid["retiring_members"][0]["private_mac"] = "00:00:00:00:00:00"
with pytest.raises(VpcError) as all_zero:
parse_vpc_delete_plan(invalid)
assert all_zero.value.error_code == "vpc_delete_manifest_invalid"
def test_delete_digest_matches_shared_php_fixture():
fixture = json.loads((
Path(__file__).with_name("fixtures") / "vpc_delete_digest.json"
).read_text(encoding="utf-8"))
plan, retiring = parse_vpc_delete_plan(fixture["document"])
assert plan.desired_digest == fixture["expected_digest"]
assert plan.desired_digest == (
"2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314"
)
assert [host.host_id for host in plan.hosts] == ["11", "2"]
assert [member.member_id for member in retiring] == ["1" * 32]
def test_root_delete_recovers_required_host_after_restart():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-recover-0001",
desired_revision=2,
retiring_members=(retiring_member_a(),),
)
async def interrupted(plan, record, gate, previous_plan=None, **kwargs):
# Simulate a process loss after retiring-member cleanup consumed its
# tombstone but before a host absence proof became durable.
for host in record["hosts"]:
host["retiring_members"] = []
coordinator._reconcile_dataplane = interrupted
with pytest.raises(VpcError) as pending:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert pending.value.status == 503
assert pending.value.error_code == "vpc_delete_incomplete"
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["delete_target"]["required_host_ids"] == ["host-a"]
assert durable["hosts"][0]["retiring_members"] == []
coordinator.recover()
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append({
"retiring": record["hosts"][0].get("retiring"),
"retiring_members": list(
record["hosts"][0].get("retiring_members", [])),
})
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert result["deleted"] is True
assert result["readback"] == "confirmed"
assert calls == [{
"retiring": True,
"retiring_members": [target["retiring_members"][0]],
}]
def test_root_delete_restart_retries_only_hosts_without_exact_removal_proof():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a", "host-b")))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-partial-restart-0001",
desired_revision=2,
hosts=("host-a", "host-b"),
retiring_members=(retiring_member_a(),),
)
calls = []
async def partial_then_confirm(
plan, record, gate, previous_plan=None, **kwargs):
host_ids = [host["host_id"] for host in record["hosts"]]
calls.append(host_ids)
if len(calls) == 1:
assert host_ids == ["host-a", "host-b"]
host_a = next(
host for host in record["hosts"]
if host["host_id"] == "host-a")
confirm_delete_hosts(plan, {"hosts": [host_a]})
host_b = next(
host for host in record["hosts"]
if host["host_id"] == "host-b")
host_b.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_host_unavailable",
})
return
assert host_ids == ["host-b"]
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = partial_then_confirm
with pytest.raises(VpcError) as incomplete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert incomplete.value.error_code == "vpc_delete_incomplete"
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
durable_hosts = {host["host_id"]: host for host in durable["hosts"]}
assert coordinator._confirmed_removal(
durable_hosts["host-a"], 2, target["desired_digest"])
assert not coordinator._confirmed_removal(
durable_hosts["host-b"], 2, target["desired_digest"])
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]}
assert coordinator._confirmed_removal(
recovered_hosts["host-a"], 2, target["desired_digest"])
assert recovered_hosts["host-b"]["state"] == "uncertain"
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert calls == [["host-a", "host-b"], ["host-b"]]
assert result["deleted"] is True
assert result["removed_host_ids"] == ["host-a", "host-b"]
assert result["readback"] == "confirmed"
def test_delete_pending_projection_does_not_probe_confirmed_offline_host():
target = delete_payload(
operation_id="vpc-delete-pending-probe-0001",
desired_revision=2,
hosts=("host-a", "host-b"),
)
plan, _manifest = parse_vpc_delete_plan(target)
nodes = (
FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
)
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820, nodes,
"f" * 64,
)
probed = []
class RetryFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, retry_plan):
for node in retry_plan.nodes:
probed.append(node.host_id)
if node.host_id == "host-a":
raise FabricError(
"confirmed host is now offline", 502,
"fabric_host_capability_probe_failed",
)
return {
"host-b": (
SimpleNamespace(manager=object()),
{"host_id": "host-b"},
),
}
coordinator = VpcCoordinator(
MemoryStore(), RetryFabric(hosts=("host-a", "host-b")),
registry=object(),
)
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
host_digests = []
host_peer_overlays = []
def confirm_host(*args, **kwargs):
host_digests.append(args[8])
host_peer_overlays.append(kwargs["fabric_peer_overlays"])
return {
"state": "active", "removed": True,
"transport_verified": True,
"private_dataplane_verified": True,
"member_nics": [], "routes": [], "wg_allowed_ips": [],
"proxy_ips": [], "bridge": None, "bridge_owner": None,
"nft_table": None, "anti_spoof": False, "rp_filter": None,
"proxy_arp": None, "proxy_arp_pvlan": None,
"readback": "confirmed", "rollback": "none",
}
coordinator._run_host_dataplane = confirm_host
record = {
"vpc_id": plan.vpc_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"hosts": [{
"host_id": "host-b", "member_ids": [], "retiring": True,
"retiring_members": [],
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
}],
}
asyncio.run(coordinator._reconcile_dataplane_unlocked(
plan, record,
{"active": False, "available": True},
previous_plan=plan,
operation_id=plan.operation_id,
))
assert probed == ["host-b"]
assert host_digests == [fabric_plan.digest]
assert host_peer_overlays == [{
"a" * 44: "100.64.0.1/32",
"b" * 44: "100.64.0.2/32",
}]
assert coordinator._confirmed_removal(
record["hosts"][0], plan.desired_revision, plan.desired_digest)
def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready():
target = payload(
desired_revision=2, operation_id="vpc-op-config-proof-0002",
hosts=[{"host_id": "host-a", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]}],
)
plan = parse_vpc_plan(target)
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
(FabricNode(
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),),
"f" * 64,
)
class RuntimeFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, _plan):
return {
"host-a": (
SimpleNamespace(manager=object()),
{"host_id": "host-a"},
),
}
coordinator = VpcCoordinator(
MemoryStore(), RuntimeFabric(hosts=("host-a",)),
registry=object(),
)
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
coordinator._run_host_dataplane = lambda *args, **kwargs: {
"state": "applying", "transport_verified": True,
"private_dataplane_verified": False,
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"observed_revision": 0, "observed_digest": None,
"member_nics": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"config_verified": True, "xml_verified": True,
"domain_running": False, "guest_network_verified": False,
"restart_required": True,
}],
"reason_code": "vpc_private_guest_restart_required",
}
record = {
"vpc_id": plan.vpc_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"hosts": [{
"host_id": "host-a", "member_ids": ["member-b"],
"retiring_members": [retiring_member_a()],
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
}],
}
asyncio.run(coordinator._reconcile_dataplane_unlocked(
plan, record, {"active": True, "available": True},
previous_plan=plan,
))
assert record["hosts"][0]["state"] == "applying"
assert record["hosts"][0]["retiring_members"] == []
def test_reconcile_skips_exact_host_so_later_host_gets_next_budget():
target = payload(
hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
],
)
plan = parse_vpc_plan(target)
gate = {"active": True, "available": True}
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=object(),
)
record = coordinator._desired_record(
plan, gate, state="applying")
calls = []
def exact(host):
member = next(
item for item in plan.members
if item.host_id == host["host_id"])
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
host.update({
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"retiring_members": [],
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": private_mac,
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": True,
"guest_network_verified": True,
"restart_required": False,
}],
})
async def consume_one_host(_plan, execution, _gate, _previous, **kwargs):
host_ids = [host["host_id"] for host in execution["hosts"]]
calls.append(host_ids)
if len(calls) == 1:
exact(execution["hosts"][0])
execution["hosts"][1].update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"last_error_code":
"vpc_private_guest_probe_deadline_exceeded",
})
else:
assert host_ids == ["host-b"]
exact(execution["hosts"][0])
coordinator._reconcile_dataplane = consume_one_host
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
assert calls == [["host-a", "host-b"], ["host-b"]]
assert all(host["state"] == "active" for host in record["hosts"])
def test_reconcile_exact_active_hosts_return_without_periodic_probe():
target = payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
])
plan = parse_vpc_plan(target)
gate = {"active": True, "available": True}
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=object(),
)
record = coordinator._desired_record(plan, gate, state="applying")
attempted_at = {"host-a": 200, "host-b": 100}
for host in record["hosts"]:
member = next(
item for item in plan.members
if item.host_id == host["host_id"])
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
host.update({
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"retiring_members": [],
"probe_attempted_at_ns": attempted_at[host["host_id"]],
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": private_mac,
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": True,
"guest_network_verified": True,
"restart_required": False,
}],
})
calls = []
async def unexpected_probe(*args, **kwargs):
calls.append((args, kwargs))
coordinator._reconcile_dataplane = unexpected_probe
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
assert calls == []
hosts = {host["host_id"]: host for host in record["hosts"]}
assert hosts["host-a"]["state"] == "active"
assert hosts["host-b"]["state"] == "active"
assert record["state"] == "active"
assert record["next_stage_allowed"] is True
assert record["private_dataplane_verified"] is True
def test_pending_host_probe_locks_only_its_execution_host_members():
target = payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
}]},
])
plan = parse_vpc_plan(target)
gate = {"active": True, "available": True}
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=object(), vm_lock=vm_lock,
)
record = coordinator._desired_record(plan, gate, state="applying")
for host in record["hosts"]:
member = next(
item for item in plan.members
if item.host_id == host["host_id"])
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
host.update({
"host_id": member.host_id,
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"host_config_reusable": True,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"retiring_members": [],
"probe_attempted_at_ns": 200 if member.host_id == "host-a" else 100,
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": private_mac,
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": True,
"guest_network_verified": True,
"restart_required": False,
}],
})
pending = next(
host for host in record["hosts"] if host["host_id"] == "host-b")
pending.update({
"state": "applying",
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
})
pending["member_nics"][0].update({
"domain_running": False,
"guest_network_verified": False,
"guest_network_readback": "stopped",
"guest_reason_code": "vpc_private_guest_stopped",
})
async def no_host_work(*_args, **_kwargs):
return None
coordinator._reconcile_dataplane_unlocked = no_host_work
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
assert acquired == [("host-b", INSTANCE_B)]
acquired.clear()
asyncio.run(coordinator._reconcile_dataplane(
plan, record, gate, plan))
assert acquired == [
("host-a", INSTANCE_A),
("host-b", INSTANCE_B),
]
def test_instance_lock_wait_consumes_dataplane_deadline(monkeypatch):
plan = parse_vpc_plan(payload())
gate = {"active": True, "available": True}
observed_deadlines = []
@asynccontextmanager
async def slow_vm_lock(_host_id, _instance_uuid):
await asyncio.sleep(0.02)
yield
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a",)),
registry=object(), vm_lock=slow_vm_lock,
)
record = coordinator._desired_record(plan, gate, state="applying")
async def capture_deadline(*_args, request_deadline=None, **_kwargs):
observed_deadlines.append(request_deadline)
coordinator._reconcile_dataplane_unlocked = capture_deadline
monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.01)
asyncio.run(coordinator._reconcile_dataplane(
plan, record, gate, plan))
assert len(observed_deadlines) == 1
assert observed_deadlines[0] < time.monotonic()
def test_reconcile_rotates_pending_host_after_it_consumes_budget():
target = payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
])
plan = parse_vpc_plan(target)
gate = {"active": True, "available": True}
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=object(),
)
record = coordinator._desired_record(
plan, gate, state="applying")
calls = []
async def consume_budget(_plan, execution, _gate, _previous, **kwargs):
calls.append([host["host_id"] for host in execution["hosts"]])
consumer = execution["hosts"][0]
consumer["probe_attempted_at_ns"] = 100 + len(calls)
consumer.update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"last_error_code":
"vpc_private_guest_probe_deadline_exceeded",
})
coordinator._reconcile_dataplane = consume_budget
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan))
assert calls == [["host-a", "host-b"], ["host-b", "host-a"]]
def test_instance_lifecycle_target_precedes_unrelated_pending_host():
target = payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
])
plan = parse_vpc_plan(target)
gate = {"active": True, "available": True}
coordinator = VpcCoordinator(
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
registry=object(),
)
record = coordinator._desired_record(plan, gate, state="applying")
record["hosts"][1]["probe_attempted_at_ns"] = 999
calls = []
async def capture_order(_plan, execution, _gate, _previous, **kwargs):
calls.append([host["host_id"] for host in execution["hosts"]])
coordinator._reconcile_dataplane = capture_order
asyncio.run(coordinator._reconcile_pending_hosts(
plan, record, gate, plan,
private_link_states={INSTANCE_B: "down"},
))
assert calls == [["host-b", "host-a"]]
def _exact_guest_pending_host(plan):
member = plan.members[0]
return {
"host_id": member.host_id,
"member_ids": [member.member_id],
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_revision": 0,
"observed_digest": None,
"state": "applying",
"transport_verified": True,
"private_dataplane_verified": False,
"config_verified": True,
"host_config_reusable": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"retiring_members": [],
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": derive_private_mac(
plan.vpc_id, member.member_id),
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": False,
"guest_network_verified": False,
"guest_network_readback": "stopped",
"guest_reason_code": "vpc_private_guest_stopped",
"restart_required": False,
}],
}
@pytest.mark.parametrize(("live", "guest_state", "reason_code"), (
(None, "stopped", "vpc_private_guest_stopped"),
("running", "pending_restart", "vpc_private_guest_restart_required"),
("running", "unavailable", "vpc_private_guest_agent_unavailable"),
))
def test_guest_only_pending_host_uses_readback_lane_without_writes(
monkeypatch, live, guest_state, reason_code):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
plan = parse_vpc_plan(payload())
previous_host = _exact_guest_pending_host(plan)
events = []
monkeypatch.setattr(VMService, "__init__",
lambda self, *args, **kwargs: None)
def reject_ensure(*args, **kwargs):
raise AssertionError("runtime readback must not ensure private0")
monkeypatch.setattr(VMService, "ensure_private_interface", reject_ensure)
def read_pair(self, instance_uuid, **kwargs):
assert kwargs["private_seed_identity"] == {
"vpc_id": plan.vpc_id,
"member_id": plan.members[0].member_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"guest_network_generation":
kwargs["private_seed_identity"][
"guest_network_generation"],
}
assert len(kwargs["private_seed_identity"][
"guest_network_generation"]) == 64
events.append(("guest-read", instance_uuid, guest_state))
return {
"inactive": f"inactive:{instance_uuid}",
"live": (f"live:{instance_uuid}" if live else None),
"seed_verified": True,
"guest_network": {
"state": guest_state, "verified": False,
"reason_code": reason_code,
},
}
monkeypatch.setattr(VMService, "read_private_domain_pair", read_pair)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
def reject_apply(*args, **kwargs):
raise AssertionError("runtime readback must not apply host config")
monkeypatch.setattr(VpcPrivateHostAdapter, "apply", reject_apply)
def observe(self, dataplane, *, xml_by_instance=None):
events.append(("host-observe", tuple(xml_by_instance or {}),
xml_by_instance[INSTANCE_A]["guest_network"]["state"]))
pair = xml_by_instance[INSTANCE_A]
assert pair["seed_verified"] is True
assert pair["inactive"] == f"inactive:{INSTANCE_A}"
return {
"state": "applying",
"config_verified": True,
"host_config_reusable": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [],
"reason_code": reason_code,
}
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
result = coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
)
assert events == [
("guest-read", INSTANCE_A, guest_state),
("host-observe", (INSTANCE_A,), guest_state),
]
assert result["state"] == "applying"
def test_runtime_readback_gate_rejects_mutation_and_stale_config():
plan = parse_vpc_plan(payload())
host = _exact_guest_pending_host(plan)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
assert coordinator._runtime_readback_eligible(
host, plan, "host-a") is True
assert coordinator._runtime_readback_eligible(
host, plan, "host-a", {INSTANCE_A: "preserve"}) is True
assert coordinator._runtime_readback_eligible(
host, plan, "host-a", {INSTANCE_A: "down"}) is False
transport_retry = dict(host, transport_verified=False)
assert coordinator._runtime_readback_eligible(
transport_retry, plan, "host-a") is True
host_drift = dict(host, host_config_reusable=False)
assert coordinator._runtime_readback_eligible(
host_drift, plan, "host-a") is False
stale_revision = dict(host, config_observed_revision=0)
assert coordinator._runtime_readback_eligible(
stale_revision, plan, "host-a") is False
changed_members = dict(host, member_ids=[])
assert coordinator._runtime_readback_eligible(
changed_members, plan, "host-a") is False
retiring = dict(host, retiring_members=[{"member_id": "old-member"}])
assert coordinator._runtime_readback_eligible(
retiring, plan, "host-a") is False
assert coordinator._runtime_readback_eligible(
host, plan, "host-a", retiring_members=[{
"member_id": "old-member",
}]) is False
retiring_host = dict(host, retiring=True)
assert coordinator._runtime_readback_eligible(
retiring_host, plan, "host-a") is False
wrong_desired_digest = dict(host, desired_digest="0" * 64)
assert coordinator._runtime_readback_eligible(
wrong_desired_digest, plan, "host-a") is False
wrong_config_digest = dict(host, config_observed_digest="0" * 64)
assert coordinator._runtime_readback_eligible(
wrong_config_digest, plan, "host-a") is False
for field, value in (
("instance_uuid", INSTANCE_B),
("private_ip_cidr", "10.20.0.99/32"),
("private_mac", "02:00:00:00:00:01"),
("config_verified", False),
("xml_verified", False),
("seed_verified", False),
):
bad_nic = dict(host["member_nics"][0], **{field: value})
bad_host = dict(host, member_nics=[bad_nic])
assert coordinator._runtime_readback_eligible(
bad_host, plan, "host-a") is False
def test_runtime_readback_unstarted_probe_reports_deadline_not_missing_xml(
monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
plan = parse_vpc_plan(payload())
previous_host = _exact_guest_pending_host(plan)
monkeypatch.setattr(VMService, "__init__",
lambda self, *args, **kwargs: None)
monkeypatch.setattr(
VMService, "ensure_private_interface",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("deadline readback must not rewrite private0")),
)
monkeypatch.setattr(
VMService, "read_private_domain_pair",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("sub-second probe must not enter libvirt")),
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("deadline readback must not apply host config")),
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "observe",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("missing XML cannot be observed as topology")),
)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
with pytest.raises(VpcError) as raised:
coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
request_deadline=time.monotonic() + 0.5,
)
assert raised.value.error_code == (
"vpc_private_guest_probe_deadline_exceeded")
def test_readback_topology_mismatch_revokes_config_proof(monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
plan = parse_vpc_plan(payload())
previous_host = _exact_guest_pending_host(plan)
previous_host["transport_verified"] = False
events = []
monkeypatch.setattr(VMService, "__init__",
lambda self, *args, **kwargs: None)
monkeypatch.setattr(
VMService, "ensure_private_interface",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("topology retry must not ensure private0")),
)
monkeypatch.setattr(
VMService, "read_private_domain_pair",
lambda self, instance_uuid, **kwargs: {
"inactive": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"seed_verified": True,
"guest_network": {
"state": "unavailable", "verified": False,
"reason_code": "vpc_private_guest_agent_unavailable",
},
},
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply",
lambda *args, **kwargs: (_ for _ in ()).throw(
AssertionError("topology retry must not apply host config")),
)
def observe(self, dataplane, *, xml_by_instance=None):
# This is the adapter boundary that runs the real observe script's
# bridge topology guard. Model a tap appearing after the old proof.
events.append("topology-observe")
return {
"state": "degraded",
"config_verified": False,
"host_config_reusable": False,
"config_observed_revision": 0,
"config_observed_digest": None,
"member_nics": [],
"reason_code": "vpc_private_dataplane_readback_mismatch",
}
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
observed = coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
)
assert events == ["topology-observe"]
assert observed["config_verified"] is False
assert observed["host_config_reusable"] is False
previous_host.update(observed)
assert coordinator._runtime_readback_eligible(
previous_host, plan, "host-a") is False
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply",
lambda self, dataplane: events.append(("apply", dataplane.host_id)),
)
def ensure(self, instance_uuid, **kwargs):
events.append(("ensure", instance_uuid))
return {
"xml": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"seed_verified": True,
"restart_required": False,
}
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
)
assert events == [
"topology-observe",
("apply", "host-a"),
("ensure", INSTANCE_A),
"topology-observe",
]
def test_readback_seed_mismatch_revokes_proof_then_retries_write(monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
plan = parse_vpc_plan(payload())
previous_host = _exact_guest_pending_host(plan)
events = []
writes_allowed = False
monkeypatch.setattr(VMService, "__init__",
lambda self, *args, **kwargs: None)
def apply(self, dataplane):
assert writes_allowed is True
events.append(("apply", dataplane.host_id))
def ensure(self, instance_uuid, **kwargs):
assert writes_allowed is True
events.append(("ensure", instance_uuid))
return {
"xml": f"inactive:{instance_uuid}", "live": None,
"seed_verified": True, "restart_required": False,
}
def read_pair(self, instance_uuid, **kwargs):
assert writes_allowed is False
assert kwargs["private_seed_identity"]["desired_digest"] == (
plan.desired_digest)
events.append(("seed-read", instance_uuid))
return {
"inactive": f"inactive:{instance_uuid}", "live": None,
"seed_verified": False,
"guest_network": {
"state": "stopped", "verified": False,
"reason_code": "vpc_private_guest_stopped",
},
}
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
monkeypatch.setattr(VMService, "read_private_domain_pair", read_pair)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(VpcPrivateHostAdapter, "apply", apply)
def observe(self, dataplane, *, xml_by_instance=None):
seed_verified = xml_by_instance[INSTANCE_A]["seed_verified"] is True
events.append(("observe", seed_verified))
return {
"state": "applying",
"config_verified": seed_verified,
"host_config_reusable": True,
"config_observed_revision": (
plan.desired_revision if seed_verified else 0),
"config_observed_digest": (
plan.desired_digest if seed_verified else None),
"member_nics": [{
**previous_host["member_nics"][0],
"seed_verified": seed_verified,
"config_verified": seed_verified,
"xml_verified": seed_verified,
}],
"reason_code": "vpc_private_guest_restart_required",
}
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
args = (
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan,
)
observed = coordinator._run_host_dataplane(
*args, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
)
assert events == [("seed-read", INSTANCE_A), ("observe", False)]
assert observed["config_verified"] is False
previous_host.update(observed)
assert coordinator._runtime_readback_eligible(
previous_host, plan, "host-a") is False
writes_allowed = True
coordinator._run_host_dataplane(
*args, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
previous_host=previous_host,
)
assert events == [
("seed-read", INSTANCE_A), ("observe", False),
("apply", "host-a"), ("ensure", INSTANCE_A), ("observe", True),
]
def test_explicit_private_link_change_keeps_full_write_lane(monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
plan = parse_vpc_plan(payload())
previous_host = _exact_guest_pending_host(plan)
events = []
monkeypatch.setattr(VMService, "__init__",
lambda self, *args, **kwargs: None)
def ensure(self, instance_uuid, **kwargs):
events.append(("ensure", instance_uuid, kwargs["link_state"]))
return {
"xml": f"inactive:{instance_uuid}", "live": None,
"seed_verified": True, "restart_required": False,
}
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply",
lambda self, dataplane: events.append(("apply", dataplane.host_id)),
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "observe",
lambda self, dataplane, **kwargs: {
"state": "applying",
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [],
"reason_code": "vpc_private_guest_stopped",
},
)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
private_link_states={INSTANCE_A: "down"},
previous_host=previous_host,
)
assert events == [
("apply", "host-a"),
("ensure", INSTANCE_A, "down"),
]
def test_guest_probe_pool_is_rolling_bounded_and_deadline_fail_closed(
monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
members = [{
"member_id": f"member-{index}",
"instance_uuid":
f"00000000-0000-4000-8000-{index + 1:012x}",
"private_ip_cidr": f"10.20.0.{index + 10}/32",
"subnet_id": "subnet-a",
} for index in range(10)]
plan = parse_vpc_plan(payload(hosts=[{
"host_id": "host-a", "members": members,
}]))
monkeypatch.setattr("virsh_panel.vpc.VPC_GUEST_PROBE_WORKERS", 2)
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
def configure(self, instance_uuid, **kwargs):
return {
"xml": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"restart_required": False,
}
monkeypatch.setattr(VMService, "ensure_private_interface", configure)
lock = threading.Lock()
active = 0
maximum = 0
started = []
def probe(self, instance_uuid, **kwargs):
nonlocal active, maximum
assert isinstance(kwargs.get("guest_probe_deadline"), float)
with lock:
active += 1
maximum = max(maximum, active)
started.append(instance_uuid)
time.sleep(0.1)
with lock:
active -= 1
return {
"inactive": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"guest_network": {
"state": "confirmed", "verified": True,
"source": "qemu_guest_agent",
},
}
monkeypatch.setattr(VMService, "read_private_domain_pair", probe)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply", lambda self, dataplane: {})
captured = {}
def observe(self, dataplane, *, xml_by_instance=None):
captured.update(xml_by_instance or {})
return {
"state": "applying",
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [],
"reason_code": "vpc_private_guest_probe_deadline_exceeded",
}
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
result = coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
request_deadline=time.monotonic() + 1.05,
)
assert maximum == 2
assert len(started) == 2
assert active == 0
assert len(captured) == 10
deadline_rows = [
pair["guest_network"] for pair in captured.values()
if pair["guest_network"].get("reason_code")
== "vpc_private_guest_probe_deadline_exceeded"
]
assert len(deadline_rows) == 8
assert result["config_verified"] is True
assert result["private_dataplane_verified"] is False
def test_guest_probe_default_pool_bounds_libvirt_reply_amplification():
from virsh_panel.vpc import VPC_GUEST_PROBE_WORKERS
# virDomainQemuAgentCommand materializes libvirt's bounded raw reply before
# Python can enforce its 192 KiB parse limit. Keep the default narrow even
# though individual tests may monkeypatch it lower for deterministic timing.
assert VPC_GUEST_PROBE_WORKERS == 4
def test_guest_probe_limit_is_process_wide_across_vpc_requests(monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
def make_plan(label, second_octet, first_uuid):
members = [{
"member_id": f"member-{label}-{index}",
"instance_uuid":
f"00000000-0000-4000-8000-{first_uuid + index:012x}",
"private_ip_cidr":
f"10.{second_octet}.0.{index + 10}/32",
"subnet_id": f"subnet-{label}",
} for index in range(4)]
return parse_vpc_plan(payload(
vpc_id=f"vpc-hk-{label}",
cidr=f"10.{second_octet}.0.0/24",
operation_id=f"vpc-op-{label}-00000001",
subnets=[{
"subnet_id": f"subnet-{label}",
"cidr": f"10.{second_octet}.0.0/25",
}],
hosts=[{"host_id": "host-a", "members": members}],
))
plans = (make_plan("a", 20, 1), make_plan("b", 21, 101))
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
monkeypatch.setattr(
VMService, "ensure_private_interface",
lambda self, instance_uuid, **kwargs: {
"xml": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"restart_required": False,
},
)
lock = threading.Lock()
release_full_wave = threading.Event()
active = 0
maximum = 0
started = 0
def probe(self, instance_uuid, **kwargs):
nonlocal active, maximum, started
with lock:
active += 1
started += 1
maximum = max(maximum, active)
if active >= 8:
release_full_wave.set()
# Without the process gate, both four-worker request pools enter
# together. With it, a bounded wave of four leaves before the next.
release_full_wave.wait(0.2)
with lock:
active -= 1
return {
"inactive": f"inactive:{instance_uuid}",
"live": f"live:{instance_uuid}",
"guest_network": {
"state": "confirmed", "verified": True,
"source": "qemu_guest_agent",
},
}
monkeypatch.setattr(VMService, "read_private_domain_pair", probe)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
monkeypatch.setattr(
VpcPrivateHostAdapter, "apply", lambda self, dataplane: {})
def observe(self, dataplane, *, xml_by_instance=None):
return {
"state": "active",
"config_verified": True,
"config_observed_revision": dataplane.vpc.desired_revision,
"config_observed_digest": dataplane.vpc.desired_digest,
"member_nics": [],
}
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
def run(plan):
return VpcCoordinator(
MemoryStore(), FakeFabric(),
)._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
request_deadline=time.monotonic() + 3,
)
with ThreadPoolExecutor(max_workers=2) as requests:
results = [future.result() for future in (
requests.submit(run, plans[0]),
requests.submit(run, plans[1]),
)]
assert started == 8
assert maximum == 4
assert all(result["config_verified"] is True for result in results)
def test_request_probe_deadline_stops_next_host_and_keeps_config_proof(
monkeypatch):
target = payload(hosts=[
{"host_id": "host-a", "members": [{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]},
{"host_id": "host-b", "members": [{
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}]},
])
plan = parse_vpc_plan(target)
fabric_plan = FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
(
FabricNode(
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
FabricNode(
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
),
"f" * 64,
)
class DeadlineFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, _plan):
return {
host_id: (
SimpleNamespace(manager=object()),
{"host_id": host_id},
)
for host_id in ("host-a", "host-b")
}
coordinator = VpcCoordinator(
MemoryStore(), DeadlineFabric(hosts=("host-a", "host-b")),
registry=object(),
)
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._fabric_record = lambda _plan: {
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.05)
calls = []
def slow_host(*args, **kwargs):
host_id = args[1]
calls.append(host_id)
time.sleep(0.08)
return {
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [{
"member_id": "member-a",
"guest_network_verified": True,
}],
"reason_code": None,
}
coordinator._run_host_dataplane = slow_host
gate = {"active": True, "available": True}
record = coordinator._desired_record(
plan, gate, state="applying")
asyncio.run(coordinator._reconcile_dataplane_unlocked(
plan, record, gate, previous_plan=plan))
assert calls == ["host-a"]
first, second = record["hosts"]
assert first["transport_verified"] is True
assert first["probe_attempted_at_ns"] > 0
assert first["config_verified"] is True
assert first["config_observed_revision"] == plan.desired_revision
assert first["private_dataplane_verified"] is False
assert first["member_nics"][0]["guest_network_verified"] is False
assert first["last_error_code"] \
== "vpc_private_guest_probe_deadline_exceeded"
assert second["transport_verified"] is False
assert second["last_error_code"] \
== "vpc_private_guest_probe_deadline_exceeded"
def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors():
deterministic_codes = (
"vpc_private_remove_requires_stopped",
"vpc_private_remove_unsafe",
"vpc_private_interface_conflict",
"vpc_seed_missing",
"vpc_seed_readback_invalid",
"vpc_seed_network_policy_migration_required",
"vpc_private_xml_invalid",
)
for index, error_code in enumerate(deterministic_codes, start=1):
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id=f"vpc-delete-deterministic-{index:04d}",
retiring_members=(retiring_member_a(),),
)
async def blocked(plan, record, gate, previous_plan=None, **kwargs):
record["hosts"][0].update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": error_code,
})
coordinator._reconcile_dataplane = blocked
with pytest.raises(VpcError) as blocked_delete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert blocked_delete.value.status == 409
assert blocked_delete.value.error_code == error_code
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
operation_id="vpc-delete-transient-error-0001",
retiring_members=(retiring_member_a(),),
)
async def transient(plan, record, gate, previous_plan=None, **kwargs):
record["hosts"][0].update({
"state": "uncertain",
"transport_verified": False,
"private_dataplane_verified": False,
"observed_revision": 0,
"observed_digest": None,
"last_error_code": "vpc_private_remove_uncertain",
})
coordinator._reconcile_dataplane = transient
with pytest.raises(VpcError) as transient_delete:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert transient_delete.value.status == 503
assert transient_delete.value.error_code == "vpc_delete_incomplete"
@pytest.mark.parametrize("include_proven_host", [False, True])
def test_root_delete_carries_forward_exact_removed_host_without_recontact(
include_proven_host):
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
detached = payload(
desired_revision=2,
operation_id="vpc-member-detach-0001",
subnets=[], members=[], routes=[], acls=[], hosts=[],
)
# Ordinary apply models the earlier member/deployment detach. Its host
# tombstone is then completed with the exact revision/digest proof that a
# lost PHP response may leave behind.
asyncio.run(coordinator.apply(detached))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring"] is True
confirm_delete_hosts(parse_vpc_plan(detached), durable)
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
async def must_not_recontact(*args, **kwargs):
raise AssertionError("exact removed host must not be contacted again")
coordinator._reconcile_dataplane = must_not_recontact
target = delete_payload(
desired_revision=3,
operation_id=("vpc-delete-proven-included-0001"
if include_proven_host
else "vpc-delete-proven-omitted-0001"),
hosts=(("host-a",) if include_proven_host else ()),
)
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result["removed_host_ids"] == ["host-a"]
assert result["readback"] == "confirmed"
def test_root_delete_requires_every_unproven_old_host_in_target():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
target = delete_payload(
desired_revision=2,
operation_id="vpc-delete-missing-host-0001",
hosts=(),
)
with pytest.raises(VpcError) as missing:
asyncio.run(coordinator.delete("vpc-hk-1", target))
assert missing.value.status == 409
assert missing.value.error_code == "vpc_delete_host_set_mismatch"
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
def test_root_delete_without_desired_record_proves_requested_hosts_or_empty():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
calls = []
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
calls.append((
[host["host_id"] for host in record["hosts"]],
[host.get("retiring") for host in record["hosts"]],
[row["instance_uuid"] for host in record["hosts"]
for row in host.get("retiring_members", [])],
))
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
vpc_id="vpc-never-applied",
operation_id="vpc-delete-never-applied-0001",
desired_revision=3,
hosts=("host-a",),
cidr="10.21.0.0/24",
retiring_members=({
"member_id": "member-never-applied",
"instance_uuid": INSTANCE_B,
"host_id": "host-a",
"private_ip_cidr": "10.21.0.10/32",
"private_mac": derive_private_mac(
"vpc-never-applied", "member-never-applied"),
},),
)
result = asyncio.run(coordinator.delete("vpc-never-applied", target))
assert result["removed_host_ids"] == ["host-a"]
assert calls == [(["host-a"], [True], [INSTANCE_B])]
empty = delete_payload(
vpc_id="vpc-empty-never-applied",
operation_id="vpc-delete-empty-never-applied-0001",
desired_revision=1,
hosts=(),
cidr="10.22.0.0/24",
)
result = asyncio.run(coordinator.delete(
"vpc-empty-never-applied", empty))
assert result["removed_host_ids"] == []
assert len(calls) == 1
def test_delete_manifest_removes_unknown_member_nic_before_host_resources(
monkeypatch):
import sys
import types
try:
import libvirt # noqa: F401
except ModuleNotFoundError:
stub = types.ModuleType("libvirt")
stub.libvirtError = type("libvirtError", (Exception,), {})
monkeypatch.setitem(sys.modules, "libvirt", stub)
from virsh_panel.libvirt_service import VMService
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
target = delete_payload(
vpc_id="vpc-remote-only",
operation_id="vpc-delete-remote-only-0001",
desired_revision=5,
hosts=("host-a",),
cidr="10.23.0.0/24",
retiring_members=({
"member_id": "member-remote-only",
"instance_uuid": INSTANCE_B,
"host_id": "host-a",
"private_ip_cidr": "10.23.0.10/32",
"private_mac": derive_private_mac(
"vpc-remote-only", "member-remote-only"),
},),
)
plan, manifest = parse_vpc_delete_plan(target)
rows = [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": member.private_mac,
} for member in manifest]
events = []
local_key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="
remote_key = "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI="
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
def remove_nic(self, instance_uuid, **kwargs):
events.append(("nic", instance_uuid, kwargs["vpc_id"]))
return {"removed": True, "readback": "confirmed"}
monkeypatch.setattr(VMService, "remove_private_interface", remove_nic)
monkeypatch.setattr(
VpcPrivateHostAdapter, "__init__",
lambda self, manager, profile, **kwargs: None,
)
def remove_host(self, dataplane):
assert dataplane.fabric_peer_keys == frozenset({remote_key})
assert dataplane.fabric_peer_overlays == frozenset({
(remote_key, "100.64.0.3/32"),
})
events.append(("host", dataplane.host_id, dataplane.vpc.vpc_id))
return {
"state": "active", "removed": True,
"bridge": None, "bridge_owner": None, "nft_table": None,
"anti_spoof": False, "rp_filter": None,
"proxy_arp": None, "proxy_arp_pvlan": None,
"proxy_ips": [], "member_nics": [], "routes": [],
"wg_allowed_ips": [], "readback": "confirmed",
}
monkeypatch.setattr(VpcPrivateHostAdapter, "remove", remove_host)
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
observed = coordinator._run_host_dataplane(
plan, "host-a", SimpleNamespace(manager=object()),
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
"iso_pool": "iso", "default_network": "default"},
[], None, retiring_members=rows,
fabric_public_keys={"host-a": local_key, "host-b": remote_key},
fabric_peer_overlays={
local_key: "100.64.0.2/32",
remote_key: "100.64.0.3/32",
},
fabric_plan_digest="ef" * 32,
)
assert events == [
("nic", INSTANCE_B, "vpc-remote-only"),
("host", "host-a", "vpc-remote-only"),
]
assert observed["removed"] is True
assert observed["readback"] == "confirmed"
def test_root_delete_receipt_survives_purge_failure_and_keeps_timestamp():
store = MemoryStore()
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
confirm_delete_hosts(plan, record)
coordinator._reconcile_dataplane = confirm
target = delete_payload(
operation_id="vpc-delete-purge-crash-0001",
retiring_members=(retiring_member_a(),),
)
original_delete = store.delete
failed = False
def fail_once(kind, key):
nonlocal failed
if kind == "vpc_desired" and not failed:
failed = True
raise RuntimeError("crash after receipt commit")
return original_delete(kind, key)
store.delete = fail_once
with pytest.raises(RuntimeError, match="after receipt commit"):
asyncio.run(coordinator.delete("vpc-hk-1", target))
durable_receipt = json.loads(store.get(
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
first_completed_at = durable_receipt["completed_at"]
assert first_completed_at > 0
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
assert result["completed_at"] == first_completed_at
assert result["readback"] == "confirmed"
assert store.get("vpc_desired", "vpc-hk-1") is None