You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4457 lines
166 KiB
Python
4457 lines
166 KiB
Python
"""Structured VPC control-plane tests (no host data-plane commands)."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import threading
|
|
import time
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.fabric import (
|
|
FabricCoordinator,
|
|
FabricError,
|
|
FabricNode,
|
|
FabricPlan,
|
|
parse_fabric_plan,
|
|
)
|
|
from virsh_panel.vpc import (
|
|
VPC_DELETE_RECEIPT_KIND,
|
|
VPC_GUEST_EXEC_KIND,
|
|
VPC_SCHEMA_VERSION,
|
|
VpcCoordinator,
|
|
VpcError,
|
|
_plan_from_record,
|
|
parse_vpc_delete_plan,
|
|
parse_vpc_plan,
|
|
)
|
|
from virsh_panel.vpc_dataplane import derive_private_mac
|
|
|
|
|
|
INSTANCE_A = "11111111-1111-4111-8111-111111111111"
|
|
INSTANCE_B = "22222222-2222-4222-8222-222222222222"
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.values = {}
|
|
self.lock = threading.RLock()
|
|
|
|
def get(self, kind, key):
|
|
with self.lock:
|
|
return self.values.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
with self.lock:
|
|
self.values[kind, key] = value
|
|
|
|
def delete(self, kind, key):
|
|
with self.lock:
|
|
self.values.pop((kind, key), None)
|
|
|
|
def list(self, kind):
|
|
with self.lock:
|
|
return {key: value for (row_kind, key), value in self.values.items()
|
|
if row_kind == kind}
|
|
|
|
|
|
class FakeFabric:
|
|
def __init__(self, *, state="active", gate=True, revision=7,
|
|
hosts=("host-a", "host-b"), endpoint_ips=None):
|
|
self.state = state
|
|
self.gate = gate
|
|
self.revision = revision
|
|
self.hosts = hosts
|
|
self.endpoint_ips = endpoint_ips or tuple(
|
|
f"203.0.113.{index + 1}" for index, _host in enumerate(hosts))
|
|
|
|
def get_status(self, fabric_id):
|
|
if fabric_id == "missing":
|
|
raise FabricError("missing", 404, "fabric_not_found")
|
|
return {
|
|
"fabric_id": fabric_id,
|
|
"region": "hk",
|
|
"revision": self.revision,
|
|
"state": self.state,
|
|
"next_stage_allowed": self.gate,
|
|
"nodes": [
|
|
{"host_id": host,
|
|
"endpoint": f"{self.endpoint_ips[index]}:51820"}
|
|
for index, host in enumerate(self.hosts)
|
|
],
|
|
}
|
|
|
|
|
|
def test_reconcile_loop_preserves_and_coalesces_wakeups_during_a_pass():
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
|
|
async def exercise():
|
|
entered = asyncio.Event()
|
|
release = asyncio.Event()
|
|
completed = asyncio.Queue()
|
|
calls = 0
|
|
|
|
async def reconcile_all():
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 1:
|
|
entered.set()
|
|
await release.wait()
|
|
completed.put_nowait(calls)
|
|
|
|
coordinator.reconcile_all = reconcile_all
|
|
task = asyncio.create_task(coordinator.reconcile_loop(interval=3600))
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), timeout=1)
|
|
coordinator.wake_reconcile()
|
|
coordinator.wake_reconcile()
|
|
release.set()
|
|
assert await asyncio.wait_for(completed.get(), timeout=1) == 1
|
|
assert await asyncio.wait_for(completed.get(), timeout=1) == 2
|
|
await asyncio.sleep(0)
|
|
assert calls == 2
|
|
coordinator.wake_reconcile()
|
|
assert await asyncio.wait_for(completed.get(), timeout=1) == 3
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
asyncio.run(exercise())
|
|
|
|
|
|
def test_reconcile_loop_keeps_periodic_fallback_without_wakeups():
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
|
|
async def exercise():
|
|
completed = asyncio.Queue()
|
|
|
|
async def reconcile_all():
|
|
completed.put_nowait(True)
|
|
|
|
coordinator.reconcile_all = reconcile_all
|
|
task = asyncio.create_task(coordinator.reconcile_loop(interval=0.01))
|
|
try:
|
|
assert await asyncio.wait_for(completed.get(), timeout=1) is True
|
|
assert await asyncio.wait_for(completed.get(), timeout=1) is True
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
asyncio.run(exercise())
|
|
|
|
|
|
def payload(**overrides):
|
|
value = {
|
|
"schema_version": VPC_SCHEMA_VERSION,
|
|
"vpc_id": "vpc-hk-1",
|
|
"region": "hk",
|
|
"fabric_id": "fabric-hk",
|
|
"fabric_revision": 7,
|
|
"cidr": "10.20.0.0/24",
|
|
"desired_revision": 1,
|
|
"operation_id": "vpc-op-00000001",
|
|
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
|
|
"hosts": [{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
|
|
]}],
|
|
"routes": [],
|
|
"acls": [],
|
|
}
|
|
value.update(overrides)
|
|
return value
|
|
|
|
|
|
def delete_payload(*, vpc_id="vpc-hk-1", desired_revision=2,
|
|
operation_id="vpc-delete-00000001",
|
|
hosts=("host-a",), retiring_members=(), **overrides):
|
|
value = payload(
|
|
vpc_id=vpc_id,
|
|
desired_revision=desired_revision,
|
|
operation_id=operation_id,
|
|
subnets=[], members=[], routes=[], acls=[],
|
|
hosts=[{"host_id": host_id, "members": []} for host_id in hosts],
|
|
)
|
|
value["retiring_members"] = list(retiring_members)
|
|
value.update(overrides)
|
|
value["desired_digest"] = parse_vpc_delete_plan(value)[0].desired_digest
|
|
return value
|
|
|
|
|
|
def confirm_delete_hosts(plan, record):
|
|
"""Fill the exact bounded absence proof required for a delete receipt."""
|
|
for host in record.get("hosts", []):
|
|
host.update({
|
|
"state": "active",
|
|
"removed": True,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"desired_revision": plan.desired_revision,
|
|
"observed_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_digest": plan.desired_digest,
|
|
"member_ids": [],
|
|
"retiring_members": [],
|
|
"member_nics": [],
|
|
"routes": [],
|
|
"wg_allowed_ips": [],
|
|
"proxy_ips": [],
|
|
"bridge": None,
|
|
"bridge_owner": None,
|
|
"nft_table": None,
|
|
"anti_spoof": False,
|
|
"rp_filter": None,
|
|
"proxy_arp": None,
|
|
"proxy_arp_pvlan": None,
|
|
"readback": "confirmed",
|
|
"rollback": "none",
|
|
})
|
|
|
|
|
|
def save_fabric_catalog(store, plan, *, plan_digest=None):
|
|
"""Persist the minimal digest-verified Fabric catalog shape."""
|
|
store.set("fabric_desired", plan.fabric_id, json.dumps({
|
|
"schema_version": 1,
|
|
"fabric_id": plan.fabric_id,
|
|
"region": plan.region,
|
|
"overlay_cidr": plan.overlay_cidr,
|
|
"revision": plan.revision,
|
|
"listen_port": plan.listen_port,
|
|
"plan_digest": plan.digest if plan_digest is None else plan_digest,
|
|
"nodes": [
|
|
{
|
|
"host_id": node.host_id,
|
|
"overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint,
|
|
"listen_port": node.listen_port,
|
|
}
|
|
for node in plan.nodes
|
|
],
|
|
}))
|
|
|
|
|
|
def retiring_member_a(*, host_id="host-a"):
|
|
return {
|
|
"member_id": "member-a",
|
|
"instance_uuid": INSTANCE_A,
|
|
"host_id": host_id,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
|
|
}
|
|
|
|
|
|
def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state():
|
|
first = parse_vpc_plan(payload())
|
|
second = parse_vpc_plan(payload(
|
|
hosts=None,
|
|
members=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip": "10.20.0.10",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
))
|
|
assert first.desired_digest == second.desired_digest
|
|
assert first.members[0].private_ip == "10.20.0.10"
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(cidr="192.0.2.0/24"))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(cidr="10.20.0.0/24",
|
|
network_cidr="10.21.0.0/24"))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(subnets=[
|
|
{"subnet_id": "a", "cidr": "10.20.0.0/25"},
|
|
{"subnet_id": "b", "cidr": "10.20.0.64/26"},
|
|
]))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(acls=[{
|
|
"acl_id": "public", "direction": "ingress", "action": "allow",
|
|
"protocol": "tcp", "source_cidr": "0.0.0.0/0",
|
|
"destination_cidr": "10.20.0.10/32", "priority": 1,
|
|
}]))
|
|
|
|
|
|
def test_member_uuid_matches_existing_libvirt_identity_boundary():
|
|
# Existing panel/libvirt records only require the canonical UUID shape;
|
|
# operator-supplied UUIDs may use non-RFC version/variant nibbles. The
|
|
# engine must accept those persisted VM identities in a VPC DTO too.
|
|
legacy_uuid = "11111111-1111-6111-0111-111111111111"
|
|
parsed = parse_vpc_plan(payload(
|
|
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
|
|
members=[{
|
|
"member_id": "member-a", "instance_uuid": legacy_uuid,
|
|
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
))
|
|
assert parsed.members[0].instance_uuid == legacy_uuid
|
|
|
|
|
|
def test_digest_matches_php_wire_shape_for_host_member_grouping():
|
|
wire = payload(
|
|
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
|
|
members=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
)
|
|
# PHP computes desired_digest before attaching operation_id to the
|
|
# signed/replayed payload; operation_id is an envelope field, not part of
|
|
# desired document identity.
|
|
digest_wire = {key: value for key, value in wire.items()
|
|
if key != "operation_id"}
|
|
expected = __import__("hashlib").sha256(
|
|
json.dumps(digest_wire, sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=True).encode("utf-8")
|
|
).hexdigest()
|
|
parsed = parse_vpc_plan(wire)
|
|
assert parsed.desired_digest == expected
|
|
assert parsed.canonical()["hosts"] == [{
|
|
"host_id": "host-a", "members": ["member-a"]
|
|
}]
|
|
|
|
|
|
def test_plan_validates_routes_acl_ports_and_member_scope():
|
|
parsed = parse_vpc_plan(payload(
|
|
hosts=[
|
|
{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
|
|
]},
|
|
{"host_id": "host-b", "members": [
|
|
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
|
|
]},
|
|
],
|
|
routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32",
|
|
"next_hop_member_id": "member-a", "metric": 20}],
|
|
acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow",
|
|
"protocol": "tcp", "source_cidr": "10.20.0.0/24",
|
|
"destination_cidr": "10.20.0.10/32", "port_from": 443,
|
|
"port_to": 443, "priority": 1}],
|
|
))
|
|
assert parsed.host_ids == {"host-a", "host-b"}
|
|
assert parsed.routes[0].next_hop_member_id == "member-a"
|
|
exact = parse_vpc_plan(payload(routes=[{
|
|
"route_id": "exact", "destination_cidr": "10.20.0.10/32",
|
|
"next_hop_member_id": "member-a",
|
|
}]))
|
|
assert exact.routes[0].destination_cidr == "10.20.0.10/32"
|
|
high_priority = parse_vpc_plan(payload(acls=[{
|
|
"acl_id": "high", "direction": "ingress", "action": "allow",
|
|
"protocol": "any", "source_cidr": "10.20.0.0/24",
|
|
"destination_cidr": "10.20.0.10/32", "priority": 1_000_000,
|
|
}]))
|
|
assert high_priority.acls[0].priority == 1_000_000
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(routes=[{
|
|
"route_id": "bad", "destination_cidr": "0.0.0.0/0",
|
|
"next_hop_host_id": "host-a",
|
|
}]))
|
|
with pytest.raises(VpcError) as broad_route:
|
|
parse_vpc_plan(payload(routes=[{
|
|
"route_id": "broad", "destination_cidr": "10.20.0.128/25",
|
|
"next_hop_member_id": "member-a",
|
|
}]))
|
|
assert broad_route.value.error_code == "vpc_route_not_exact"
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(acls=[{
|
|
"acl_id": "bad", "direction": "egress", "action": "allow",
|
|
"protocol": "icmp", "source_cidr": "10.20.0.0/24",
|
|
"destination_cidr": "10.20.0.0/24", "port_from": 1,
|
|
}]))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(routes=[{
|
|
"route_id": "alias-conflict",
|
|
"destination_cidr": "10.20.0.0/25",
|
|
"destination": "10.20.0.128/25",
|
|
"next_hop_member_id": "member-a",
|
|
}]))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(hosts=[{
|
|
"host_id": "host-a", "members": ["member-a", "member-a"],
|
|
}]))
|
|
with pytest.raises(VpcError):
|
|
parse_vpc_plan(payload(
|
|
hosts=[],
|
|
members=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
))
|
|
|
|
|
|
def test_coordinator_persists_desired_children_and_blocks_without_fabric():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False))
|
|
result = asyncio.run(coordinator.apply(payload()))
|
|
assert result["state"] == "degraded"
|
|
assert result["next_stage_allowed"] is False
|
|
assert result["observed_revision"] == 0
|
|
assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False
|
|
assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"}
|
|
assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"}
|
|
assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"}
|
|
|
|
|
|
def test_guest_exec_ledger_is_immediate_bounded_private_and_restart_safe():
|
|
store = MemoryStore()
|
|
plan = parse_vpc_plan(payload())
|
|
member = plan.members[0]
|
|
coordinator = VpcCoordinator(store, FakeFabric())
|
|
state, persist, clear = coordinator._guest_exec_context(plan, member)
|
|
assert state == {}
|
|
core = {
|
|
"version": 1,
|
|
"phase": "pending",
|
|
"domain_uuid": member.instance_uuid,
|
|
"domain_runtime_id": 7,
|
|
"qemu_runtime_token": "a" * 32,
|
|
"guest_network_generation": "generation-1",
|
|
"probe_kind": "link",
|
|
"pid": 41,
|
|
}
|
|
|
|
persist(core)
|
|
|
|
key = f"{plan.vpc_id}.{member.instance_uuid}"
|
|
encoded = store.get(VPC_GUEST_EXEC_KIND, key)
|
|
assert encoded is not None
|
|
assert len(encoded.encode("utf-8")) <= 1024
|
|
assert json.loads(encoded)["pid"] == 41
|
|
restarted = VpcCoordinator(store, FakeFabric())
|
|
loaded, _persist_again, clear_again = restarted._guest_exec_context(
|
|
plan, member)
|
|
assert loaded == core
|
|
|
|
asyncio.run(coordinator.apply(payload()))
|
|
public = json.dumps(coordinator.get_status(plan.vpc_id), sort_keys=True)
|
|
assert '"pid"' not in public
|
|
assert "qemu_runtime_token" not in public
|
|
assert "a" * 32 not in public
|
|
|
|
clear_again()
|
|
assert store.get(VPC_GUEST_EXEC_KIND, key) is None
|
|
persist(core)
|
|
coordinator._purge_deleted_vpc(plan.vpc_id)
|
|
assert store.get(VPC_GUEST_EXEC_KIND, key) is None
|
|
|
|
|
|
def test_guest_exec_ledger_rejects_oversize_or_unknown_fields():
|
|
store = MemoryStore()
|
|
plan = parse_vpc_plan(payload())
|
|
member = plan.members[0]
|
|
key = f"{plan.vpc_id}.{member.instance_uuid}"
|
|
store.set(VPC_GUEST_EXEC_KIND, key, "x" * 1025)
|
|
with pytest.raises(VpcError, match="账本损坏"):
|
|
VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member)
|
|
|
|
bad = {
|
|
"vpc_id": plan.vpc_id,
|
|
"host_id": member.host_id,
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"version": 1,
|
|
"phase": "launch_intent",
|
|
"domain_uuid": member.instance_uuid,
|
|
"domain_runtime_id": 7,
|
|
"qemu_runtime_token": "a" * 32,
|
|
"guest_network_generation": "generation-1",
|
|
"probe_kind": "link",
|
|
"unexpected": True,
|
|
}
|
|
store.set(VPC_GUEST_EXEC_KIND, key, json.dumps(bad))
|
|
with pytest.raises(VpcError, match="账本损坏"):
|
|
VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member)
|
|
|
|
|
|
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral():
|
|
store = MemoryStore()
|
|
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
|
|
assert result["state"] == "applying"
|
|
assert result["next_stage_allowed"] is False
|
|
assert result["observed"]["fabric_gate"]["active"] is True
|
|
assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True
|
|
|
|
|
|
def test_recovery_reopens_only_the_control_plane_as_applying():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
asyncio.run(coordinator.apply(payload()))
|
|
coordinator.recover()
|
|
|
|
recovered = asyncio.run(coordinator.reconcile("vpc-hk-1"))
|
|
assert recovered["state"] == "applying"
|
|
assert recovered["next_stage_allowed"] is False
|
|
assert recovered["hosts"][0]["state"] == "applying"
|
|
assert recovered["hosts"][0]["transport_verified"] is True
|
|
|
|
|
|
def test_empty_vpc_cannot_open_fabric_gate():
|
|
store = MemoryStore()
|
|
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
|
|
hosts=[], members=[],
|
|
)))
|
|
assert result["state"] == "degraded"
|
|
assert result["next_stage_allowed"] is False
|
|
assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts"
|
|
|
|
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-a", "members": []}], members=[],
|
|
)))
|
|
assert result["state"] == "degraded"
|
|
assert result["fabric_gate"]["error_code"] == "vpc_no_members"
|
|
|
|
|
|
def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a", "host-b"))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
result = asyncio.run(coordinator.apply(payload(
|
|
hosts=[{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
|
|
]}],
|
|
)))
|
|
assert result["state"] == "applying"
|
|
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
digest = raw["desired_digest"]
|
|
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {"state": "active", "observed_revision": 1,
|
|
"observed_digest": digest, "transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"readback": {"safe": "ok", "oversized": "x" * 513}},
|
|
}))
|
|
assert observed["state"] == "active"
|
|
assert observed["next_stage_allowed"] is True
|
|
assert observed["hosts"][0]["readback"] == {"safe": "ok"}
|
|
fabric.revision = 8
|
|
blocked = asyncio.run(coordinator.reconcile("vpc-hk-1"))
|
|
assert blocked["state"] == "degraded"
|
|
assert blocked["next_stage_allowed"] is False
|
|
|
|
|
|
def test_transport_only_observation_never_opens_private_dataplane_gate():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
result = asyncio.run(coordinator.apply(payload()))
|
|
digest = result["desired_digest"]
|
|
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {"state": "active", "observed_revision": 1,
|
|
"observed_digest": digest, "transport_verified": True},
|
|
}))
|
|
assert pending["state"] == "applying"
|
|
assert pending["next_stage_allowed"] is False
|
|
assert pending["private_dataplane_verified"] is False
|
|
assert pending["hosts"][0]["state"] == "applying"
|
|
assert pending["hosts"][0]["last_error_code"] == (
|
|
"vpc_private_dataplane_unverified"
|
|
)
|
|
|
|
|
|
def test_partial_host_observation_is_degraded_and_never_active():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a", "host-b"))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
result = asyncio.run(coordinator.apply(payload(
|
|
hosts=[
|
|
{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
|
|
]},
|
|
{"host_id": "host-b", "members": [
|
|
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
|
|
]},
|
|
],
|
|
)))
|
|
digest = result["desired_digest"]
|
|
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {"state": "active", "observed_revision": 1,
|
|
"observed_digest": digest, "transport_verified": True,
|
|
"private_dataplane_verified": True},
|
|
"host-b": {"state": "degraded", "observed_revision": 0,
|
|
"observed_digest": None, "transport_verified": False},
|
|
}))
|
|
assert observed["state"] == "degraded"
|
|
assert observed["next_stage_allowed"] is False
|
|
with pytest.raises(VpcError) as inconsistent:
|
|
asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {"state": "active", "observed_revision": 1,
|
|
"observed_digest": digest, "transport_verified": False,
|
|
"private_dataplane_verified": True},
|
|
"host-b": {"state": "degraded", "observed_revision": 0,
|
|
"observed_digest": None, "transport_verified": False},
|
|
}))
|
|
assert inconsistent.value.error_code == "vpc_observation_invalid"
|
|
|
|
|
|
def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric()
|
|
original = fabric.get_status
|
|
|
|
def wrong_region(fabric_id):
|
|
status = original(fabric_id)
|
|
status["region"] = "us"
|
|
return status
|
|
|
|
fabric.get_status = wrong_region
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
|
|
assert error.value.error_code == "vpc_fabric_region_mismatch"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_fabric_gate_rejects_underlay_endpoint_inside_tenant_cidr():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(
|
|
hosts=("host-a",), endpoint_ips=("10.20.0.50",),
|
|
)
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
|
|
|
|
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_vpc_rejects_endpoint_from_any_other_fabric_catalog_entry():
|
|
store = MemoryStore()
|
|
other = parse_fabric_plan({
|
|
"fabric_id": "fabric-sg",
|
|
"region": "sg",
|
|
"overlay_cidr": "100.64.2.0/24",
|
|
"revision": 3,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-c",
|
|
"overlay_ip": "100.64.2.1",
|
|
"endpoint": "10.20.0.50:51820",
|
|
},
|
|
{
|
|
"host_id": "host-d",
|
|
"overlay_ip": "100.64.2.2",
|
|
"endpoint": "203.0.113.4:51820",
|
|
},
|
|
],
|
|
})
|
|
store.set("fabric_desired", other.fabric_id, json.dumps({
|
|
"fabric_id": other.fabric_id,
|
|
"region": other.region,
|
|
"overlay_cidr": other.overlay_cidr,
|
|
"revision": other.revision,
|
|
"listen_port": other.listen_port,
|
|
"plan_digest": other.digest,
|
|
"nodes": [
|
|
{
|
|
"host_id": node.host_id,
|
|
"overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint,
|
|
"listen_port": node.listen_port,
|
|
}
|
|
for node in other.nodes
|
|
],
|
|
}))
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
|
|
|
|
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_vpc_fails_closed_on_corrupt_global_fabric_catalog_entry():
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", "fabric-corrupt", "{}")
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
|
|
|
|
assert error.value.error_code == "vpc_fabric_record_corrupt"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_vpc_fails_closed_on_stale_global_fabric_plan_digest():
|
|
store = MemoryStore()
|
|
other = parse_fabric_plan({
|
|
"fabric_id": "fabric-sg",
|
|
"region": "sg",
|
|
"overlay_cidr": "100.64.2.0/24",
|
|
"revision": 3,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-c",
|
|
"overlay_ip": "100.64.2.1",
|
|
"endpoint": "203.0.113.3:51820",
|
|
},
|
|
{
|
|
"host_id": "host-d",
|
|
"overlay_ip": "100.64.2.2",
|
|
"endpoint": "203.0.113.4:51820",
|
|
},
|
|
],
|
|
})
|
|
store.set("fabric_desired", other.fabric_id, json.dumps({
|
|
"fabric_id": other.fabric_id,
|
|
"region": other.region,
|
|
"overlay_cidr": other.overlay_cidr,
|
|
"revision": other.revision,
|
|
"listen_port": other.listen_port,
|
|
"plan_digest": "0" * 64,
|
|
"nodes": [
|
|
{
|
|
"host_id": node.host_id,
|
|
"overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint,
|
|
"listen_port": node.listen_port,
|
|
}
|
|
for node in other.nodes
|
|
],
|
|
}))
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
|
|
|
|
assert error.value.error_code == "vpc_fabric_record_corrupt"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_vpc_apply_and_fabric_endpoint_change_share_catalog_admission_lock():
|
|
store = MemoryStore()
|
|
current = parse_fabric_plan({
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 7,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
store.set("fabric_desired", current.fabric_id, json.dumps({
|
|
"schema_version": 1,
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision,
|
|
"listen_port": current.listen_port,
|
|
"plan_digest": current.digest,
|
|
"state": "active",
|
|
"next_stage_allowed": True,
|
|
"public_keys": {
|
|
"host-a": "unused-for-catalog-race-a",
|
|
"host-b": "unused-for-catalog-race-b",
|
|
},
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-a",
|
|
"overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820",
|
|
"listen_port": 51820,
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
},
|
|
{
|
|
"host_id": "host-b",
|
|
"overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820",
|
|
"listen_port": 51820,
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
},
|
|
],
|
|
"updated_at": int(time.time()),
|
|
}))
|
|
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
coordinator = VpcCoordinator(
|
|
store, fabric, catalog_lock=fabric.catalog_mutation_lock)
|
|
entered = asyncio.Event()
|
|
release = asyncio.Event()
|
|
original_reconcile = coordinator._reconcile_dataplane
|
|
|
|
async def paused_reconcile(*args, **kwargs):
|
|
entered.set()
|
|
await release.wait()
|
|
return await original_reconcile(*args, **kwargs)
|
|
|
|
coordinator._reconcile_dataplane = paused_reconcile
|
|
candidate = {
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-a",
|
|
"overlay_ip": "100.64.1.1",
|
|
"endpoint": "10.20.0.50:51820",
|
|
},
|
|
{
|
|
"host_id": "host-b",
|
|
"overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820",
|
|
},
|
|
],
|
|
}
|
|
|
|
async def run_race():
|
|
vpc_task = asyncio.create_task(coordinator.apply(payload()))
|
|
await entered.wait()
|
|
fabric_task = asyncio.create_task(fabric.apply(candidate))
|
|
await asyncio.sleep(0)
|
|
assert fabric_task.done() is False
|
|
release.set()
|
|
vpc_result = await vpc_task
|
|
with pytest.raises(FabricError) as error:
|
|
await fabric_task
|
|
return vpc_result, error.value
|
|
|
|
result, error = asyncio.run(run_race())
|
|
|
|
assert result["fabric_gate"]["active"] is True
|
|
assert error.error_code == "fabric_endpoint_vpc_conflict"
|
|
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert saved["cidr"] == "10.20.0.0/24"
|
|
|
|
|
|
def test_fabric_first_endpoint_change_blocks_conflicting_vpc_admission():
|
|
store = MemoryStore()
|
|
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
gate_fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(
|
|
store, gate_fabric,
|
|
catalog_lock=lock_owner.catalog_mutation_lock,
|
|
)
|
|
entered = asyncio.Event()
|
|
release = asyncio.Event()
|
|
|
|
async def publish_endpoint(_plan):
|
|
entered.set()
|
|
await release.wait()
|
|
gate_fabric.endpoint_ips = ("10.20.0.50",)
|
|
return {"state": "active"}
|
|
|
|
lock_owner._apply_locked = publish_endpoint
|
|
candidate = {
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-a",
|
|
"overlay_ip": "100.64.1.1",
|
|
"endpoint": "10.20.0.50:51820",
|
|
},
|
|
{
|
|
"host_id": "host-b",
|
|
"overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820",
|
|
},
|
|
],
|
|
}
|
|
|
|
async def run_race():
|
|
fabric_task = asyncio.create_task(lock_owner.apply(candidate))
|
|
await entered.wait()
|
|
vpc_task = asyncio.create_task(coordinator.apply(payload()))
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
release.set()
|
|
await fabric_task
|
|
with pytest.raises(VpcError) as error:
|
|
await vpc_task
|
|
return error.value
|
|
|
|
error = asyncio.run(run_race())
|
|
|
|
assert error.error_code == "vpc_fabric_endpoint_overlap"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_cancelled_fabric_apply_drains_thread_before_releasing_catalog_lock():
|
|
store = MemoryStore()
|
|
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
|
|
async def blocked_apply(_plan):
|
|
def mutate():
|
|
entered.set()
|
|
assert release.wait(timeout=2)
|
|
store.set("test", "fabric-thread", "settled")
|
|
|
|
await asyncio.to_thread(mutate)
|
|
return {"state": "settled"}
|
|
|
|
fabric._apply_locked = blocked_apply
|
|
candidate = {
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-a",
|
|
"overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820",
|
|
},
|
|
{
|
|
"host_id": "host-b",
|
|
"overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820",
|
|
},
|
|
],
|
|
}
|
|
|
|
async def run_cancel():
|
|
fabric_task = asyncio.create_task(fabric.apply(candidate))
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert entered.is_set()
|
|
vpc_task = asyncio.create_task(coordinator.apply(payload()))
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
fabric_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert fabric_task.done() is False
|
|
assert vpc_task.done() is False
|
|
assert store.get("test", "fabric-thread") is None
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await fabric_task
|
|
return await asyncio.wait_for(vpc_task, timeout=1)
|
|
|
|
result = asyncio.run(run_cancel())
|
|
|
|
assert store.get("test", "fabric-thread") == "settled"
|
|
assert result["vpc_id"] == "vpc-hk-1"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is not None
|
|
|
|
|
|
def test_cancelled_vpc_apply_drains_thread_and_saves_before_fabric_enters():
|
|
store = MemoryStore()
|
|
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(), catalog_lock=fabric.catalog_mutation_lock)
|
|
vpc_thread_entered = threading.Event()
|
|
vpc_thread_release = threading.Event()
|
|
fabric_entered = asyncio.Event()
|
|
|
|
async def blocked_reconcile(_plan, record, _gate, _previous_plan):
|
|
def mutate():
|
|
vpc_thread_entered.set()
|
|
assert vpc_thread_release.wait(timeout=2)
|
|
record["thread_mutation_settled"] = True
|
|
|
|
await asyncio.to_thread(mutate)
|
|
|
|
async def observed_fabric_apply(_plan):
|
|
fabric_entered.set()
|
|
return {"state": "settled"}
|
|
|
|
coordinator._reconcile_dataplane = blocked_reconcile
|
|
fabric._apply_locked = observed_fabric_apply
|
|
candidate = {
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{
|
|
"host_id": "host-a",
|
|
"overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820",
|
|
},
|
|
{
|
|
"host_id": "host-b",
|
|
"overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820",
|
|
},
|
|
],
|
|
}
|
|
|
|
async def run_cancel():
|
|
vpc_task = asyncio.create_task(coordinator.apply(payload()))
|
|
for _attempt in range(100):
|
|
if vpc_thread_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert vpc_thread_entered.is_set()
|
|
fabric_task = asyncio.create_task(fabric.apply(candidate))
|
|
await asyncio.sleep(0)
|
|
assert fabric_entered.is_set() is False
|
|
vpc_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
assert fabric_entered.is_set() is False
|
|
prewrite = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
expected = parse_vpc_plan(payload())
|
|
assert prewrite["desired_revision"] == expected.desired_revision
|
|
assert prewrite["desired_digest"] == expected.desired_digest
|
|
assert "thread_mutation_settled" not in prewrite
|
|
vpc_thread_release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await vpc_task
|
|
await asyncio.wait_for(fabric_task, timeout=1)
|
|
|
|
asyncio.run(run_cancel())
|
|
|
|
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert saved["thread_mutation_settled"] is True
|
|
|
|
|
|
def test_cancelled_root_delete_drains_host_mutation_before_catalog_unlock():
|
|
store = MemoryStore()
|
|
fabric = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)),
|
|
catalog_lock=fabric.catalog_mutation_lock,
|
|
)
|
|
delete_thread_entered = threading.Event()
|
|
delete_thread_release = threading.Event()
|
|
fabric_entered = asyncio.Event()
|
|
|
|
async def blocked_delete(plan, record, _gate, _previous_plan, **_kwargs):
|
|
def mutate():
|
|
delete_thread_entered.set()
|
|
assert delete_thread_release.wait(timeout=2)
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
await asyncio.to_thread(mutate)
|
|
|
|
async def observed_fabric_apply(_plan):
|
|
fabric_entered.set()
|
|
return {"state": "settled"}
|
|
|
|
coordinator._reconcile_dataplane = blocked_delete
|
|
fabric._apply_locked = observed_fabric_apply
|
|
target = delete_payload(
|
|
vpc_id="vpc-never-applied",
|
|
desired_revision=3,
|
|
operation_id="vpc-delete-cancel-drain-0001",
|
|
cidr="10.21.0.0/24",
|
|
)
|
|
candidate = {
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
}
|
|
|
|
async def run_cancel():
|
|
delete_task = asyncio.create_task(coordinator.delete(
|
|
"vpc-never-applied", target))
|
|
for _attempt in range(100):
|
|
if delete_thread_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert delete_thread_entered.is_set()
|
|
# The host-bearing absent-target delete has already published its
|
|
# temporary CIDR reservation, but still owns the shared catalog lock.
|
|
assert store.get("vpc_desired", "vpc-never-applied") is not None
|
|
fabric_task = asyncio.create_task(fabric.apply(candidate))
|
|
await asyncio.sleep(0)
|
|
assert fabric_entered.is_set() is False
|
|
|
|
delete_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert delete_task.done() is False
|
|
assert fabric_entered.is_set() is False
|
|
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None
|
|
|
|
delete_thread_release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await delete_task
|
|
await asyncio.wait_for(fabric_task, timeout=1)
|
|
|
|
asyncio.run(run_cancel())
|
|
|
|
receipt = json.loads(store.get(
|
|
VPC_DELETE_RECEIPT_KIND, "vpc-never-applied"))
|
|
assert receipt["state"] == "deleted"
|
|
assert receipt["readback"] == "confirmed"
|
|
assert store.get("vpc_desired", "vpc-never-applied") is None
|
|
assert fabric_entered.is_set() is True
|
|
|
|
|
|
@pytest.mark.parametrize("entrypoint", ["observe", "reconcile", "instance"])
|
|
def test_fabric_gate_change_precedes_every_vpc_evidence_publication(
|
|
entrypoint):
|
|
store = MemoryStore()
|
|
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
gate_fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(
|
|
store, gate_fabric,
|
|
catalog_lock=lock_owner.catalog_mutation_lock,
|
|
)
|
|
initial = asyncio.run(coordinator.apply(payload()))
|
|
digest = initial["desired_digest"]
|
|
active_observation = {
|
|
"host-a": {
|
|
"state": "active",
|
|
"observed_revision": 1,
|
|
"observed_digest": digest,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"config_observed_revision": 1,
|
|
"config_observed_digest": digest,
|
|
},
|
|
}
|
|
active = asyncio.run(coordinator.observe(
|
|
"vpc-hk-1", active_observation))
|
|
assert active["next_stage_allowed"] is True
|
|
|
|
async def run_race():
|
|
fabric_entered = asyncio.Event()
|
|
release_fabric = asyncio.Event()
|
|
|
|
async def change_fabric_gate():
|
|
async with lock_owner.catalog_mutation_lock():
|
|
fabric_entered.set()
|
|
await release_fabric.wait()
|
|
gate_fabric.revision = 8
|
|
gate_fabric.state = "degraded"
|
|
gate_fabric.gate = False
|
|
|
|
fabric_task = asyncio.create_task(change_fabric_gate())
|
|
await fabric_entered.wait()
|
|
if entrypoint == "observe":
|
|
vpc_task = asyncio.create_task(coordinator.observe(
|
|
"vpc-hk-1", active_observation))
|
|
elif entrypoint == "reconcile":
|
|
vpc_task = asyncio.create_task(coordinator.reconcile("vpc-hk-1"))
|
|
else:
|
|
vpc_task = asyncio.create_task(coordinator.reconcile_instance(
|
|
INSTANCE_A,
|
|
host_id="host-a",
|
|
expected={
|
|
"required": True,
|
|
"vpc_id": "vpc-hk-1",
|
|
"member_id": "member-a",
|
|
},
|
|
))
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
release_fabric.set()
|
|
await fabric_task
|
|
return await asyncio.wait_for(vpc_task, timeout=1)
|
|
|
|
result = asyncio.run(run_race())
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
|
|
assert durable["fabric_gate"]["fabric_revision"] == 8
|
|
assert durable["fabric_gate"]["active"] is False
|
|
assert durable["next_stage_allowed"] is False
|
|
assert durable["state"] != "active"
|
|
if entrypoint == "instance":
|
|
assert result["readback"] != "confirmed"
|
|
else:
|
|
assert result["next_stage_allowed"] is False
|
|
|
|
|
|
def test_observe_locks_active_and_retiring_vm_identities_in_sorted_order():
|
|
store = MemoryStore()
|
|
acquired = []
|
|
|
|
@asynccontextmanager
|
|
async def vm_lock(host_id, instance_uuid):
|
|
acquired.append((host_id, instance_uuid))
|
|
yield
|
|
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
|
|
asyncio.run(coordinator.apply(payload(hosts=[{
|
|
"host_id": "host-a",
|
|
"members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a"},
|
|
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a"},
|
|
],
|
|
}])))
|
|
second = asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-observe-lock-order-0002",
|
|
)))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert durable["hosts"][0]["retiring_members"][0][
|
|
"instance_uuid"] == INSTANCE_B
|
|
acquired.clear()
|
|
|
|
asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {
|
|
"state": "active",
|
|
"observed_revision": 2,
|
|
"observed_digest": second["desired_digest"],
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
},
|
|
}))
|
|
|
|
assert acquired == [
|
|
("host-a", INSTANCE_A),
|
|
("host-a", INSTANCE_B),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("entrypoint", ["reconcile", "instance"])
|
|
def test_cancelled_vpc_retry_drains_host_thread_before_catalog_unlock(
|
|
entrypoint):
|
|
store = MemoryStore()
|
|
lock_owner = FabricCoordinator(store, SimpleNamespace(components={}))
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)),
|
|
catalog_lock=lock_owner.catalog_mutation_lock,
|
|
)
|
|
asyncio.run(coordinator.apply(payload()))
|
|
thread_entered = threading.Event()
|
|
thread_release = threading.Event()
|
|
|
|
async def blocked_reconcile(_plan, record, _gate, _previous_plan,
|
|
**_kwargs):
|
|
def mutate():
|
|
thread_entered.set()
|
|
assert thread_release.wait(timeout=2)
|
|
record["hosts"][0]["cancel_drain_marker"] = entrypoint
|
|
|
|
await asyncio.to_thread(mutate)
|
|
|
|
coordinator._reconcile_dataplane = blocked_reconcile
|
|
|
|
async def run_cancel():
|
|
fabric_entered = asyncio.Event()
|
|
|
|
async def observed_fabric_apply(_plan):
|
|
fabric_entered.set()
|
|
return {"state": "settled"}
|
|
|
|
lock_owner._apply_locked = observed_fabric_apply
|
|
if entrypoint == "reconcile":
|
|
vpc_task = asyncio.create_task(
|
|
coordinator.reconcile("vpc-hk-1"))
|
|
else:
|
|
vpc_task = asyncio.create_task(coordinator.reconcile_instance(
|
|
INSTANCE_A,
|
|
host_id="host-a",
|
|
expected={
|
|
"required": True,
|
|
"vpc_id": "vpc-hk-1",
|
|
"member_id": "member-a",
|
|
},
|
|
))
|
|
for _attempt in range(100):
|
|
if thread_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert thread_entered.is_set()
|
|
fabric_task = asyncio.create_task(lock_owner.apply({
|
|
"fabric_id": "fabric-hk",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 8,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
}))
|
|
await asyncio.sleep(0)
|
|
assert fabric_entered.is_set() is False
|
|
|
|
vpc_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
assert fabric_entered.is_set() is False
|
|
thread_release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await vpc_task
|
|
await asyncio.wait_for(fabric_task, timeout=1)
|
|
assert fabric_entered.is_set() is True
|
|
|
|
asyncio.run(run_cancel())
|
|
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert durable["hosts"][0]["cancel_drain_marker"] == entrypoint
|
|
|
|
|
|
def test_fabric_gate_rejects_tampered_identity_without_saving():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric()
|
|
original = fabric.get_status
|
|
|
|
def tampered(fabric_id):
|
|
status = original(fabric_id)
|
|
status["fabric_id"] = "other-fabric"
|
|
return status
|
|
|
|
fabric.get_status = tampered
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
|
|
assert error.value.error_code == "vpc_fabric_identity_mismatch"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
|
|
def test_fabric_gate_keeps_revision_mismatch_as_reconcilable_state():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric()
|
|
original = fabric.get_status
|
|
|
|
def tampered(fabric_id):
|
|
status = original(fabric_id)
|
|
status["revision"] = True
|
|
return status
|
|
|
|
fabric.get_status = tampered
|
|
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
|
|
assert result["next_stage_allowed"] is False
|
|
assert result["state"] == "degraded"
|
|
|
|
|
|
def test_same_vpc_revision_and_digest_cannot_borrow_another_operation_receipt(monkeypatch):
|
|
async def run():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
first = await coordinator.apply(payload(operation_id="a" * 32))
|
|
before = store.get("vpc_desired", "vpc-hk-1")
|
|
retries = []
|
|
|
|
async def observe_only(*_args, **_kwargs):
|
|
retries.append(True)
|
|
|
|
monkeypatch.setattr(coordinator, "_reconcile_pending_hosts", observe_only)
|
|
with pytest.raises(VpcError) as conflict:
|
|
await coordinator.apply(payload(operation_id="b" * 32))
|
|
assert conflict.value.status == 409
|
|
assert conflict.value.error_code == "vpc_revision_conflict"
|
|
assert store.get("vpc_desired", "vpc-hk-1") == before
|
|
assert retries == []
|
|
replay = await coordinator.apply(payload(operation_id="a" * 32))
|
|
assert replay["operation_id"] == first["operation_id"]
|
|
assert retries == [True]
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_coordinator_revision_conflict_and_tampered_record_fail_closed():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
first = asyncio.run(coordinator.apply(payload()))
|
|
replay = asyncio.run(coordinator.apply(payload()))
|
|
assert replay["desired_digest"] == first["desired_digest"]
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(coordinator.apply(payload(
|
|
operation_id="vpc-op-00000002",
|
|
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip": "10.20.0.12",
|
|
"subnet_id": "subnet-a"}],
|
|
hosts=[{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
|
|
]}],
|
|
)))
|
|
assert error.value.error_code == "vpc_revision_conflict"
|
|
with pytest.raises(VpcError) as invalid_revision:
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=0, operation_id="vpc-op-00000003",
|
|
)))
|
|
assert invalid_revision.value.error_code == "vpc_invalid"
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000004",
|
|
)))
|
|
with pytest.raises(VpcError) as stale:
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=1, operation_id="vpc-op-00000005",
|
|
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip": "10.20.0.12",
|
|
"subnet_id": "subnet-a"}],
|
|
hosts=[{"host_id": "host-a", "members": [
|
|
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
|
|
]}],
|
|
)))
|
|
assert stale.value.error_code == "vpc_revision_conflict"
|
|
tampered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
tampered["desired_digest"] = "0" * 64
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
|
|
with pytest.raises(VpcError) as corrupt:
|
|
asyncio.run(coordinator.reconcile("vpc-hk-1"))
|
|
assert corrupt.value.error_code == "vpc_record_corrupt"
|
|
tampered["vpc_id"] = "other-vpc"
|
|
tampered["desired_digest"] = first["desired_digest"]
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
|
|
with pytest.raises(VpcError) as mismatched:
|
|
coordinator.get_status("vpc-hk-1")
|
|
assert mismatched.value.error_code == "vpc_record_corrupt"
|
|
|
|
# Restore a structurally valid root and corrupt only observed evidence;
|
|
# a newer target must not overwrite that evidence silently.
|
|
restored = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
restored["vpc_id"] = "vpc-hk-1"
|
|
restored["desired_revision"] = 2
|
|
restored["desired_digest"] = parse_vpc_plan(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000004",
|
|
)).desired_digest
|
|
restored["observed_revision"] = 99
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(restored))
|
|
with pytest.raises(VpcError) as observed_corrupt:
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=3, operation_id="vpc-op-00000006",
|
|
)))
|
|
assert observed_corrupt.value.error_code == "vpc_record_corrupt"
|
|
|
|
|
|
def test_recover_invalidates_observed_gate_and_record_round_trips():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
raw["state"] = "active"
|
|
raw["next_stage_allowed"] = True
|
|
raw["observed_revision"] = 1
|
|
raw["observed_digest"] = raw["desired_digest"]
|
|
raw["hosts"][0].update(state="active", observed_revision=1,
|
|
observed_digest=raw["desired_digest"],
|
|
transport_verified=True,
|
|
private_dataplane_verified=True)
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
|
|
coordinator.recover()
|
|
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert recovered["state"] == "uncertain"
|
|
assert recovered["next_stage_allowed"] is False
|
|
assert recovered["observed_revision"] == 1
|
|
assert recovered["observed_digest"] == recovered["desired_digest"]
|
|
assert recovered["fabric_gate"]["active"] is False
|
|
assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending"
|
|
assert recovered["hosts"][0]["state"] == "uncertain"
|
|
assert recovered["hosts"][0]["transport_verified"] is False
|
|
assert recovered["hosts"][0]["private_dataplane_verified"] is False
|
|
assert recovered["hosts"][0]["observed_revision"] == 1
|
|
assert _plan_from_record(recovered).vpc_id == "vpc-hk-1"
|
|
|
|
|
|
def test_removed_children_are_tombstoned_until_observed_cleanup():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
changed = payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-a", "members": []}],
|
|
members=[],
|
|
)
|
|
asyncio.run(coordinator.apply(changed))
|
|
old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a"))
|
|
assert old["state"] == "removed"
|
|
assert old["removed"] is True
|
|
|
|
|
|
def test_new_revision_drops_host_with_exact_prior_removal_proof():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
first_payload = payload(
|
|
hosts=[{"host_id": "host-a", "members": []}], members=[])
|
|
asyncio.run(coordinator.apply(first_payload))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
confirm_delete_hosts(parse_vpc_plan(first_payload), durable)
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
|
|
|
|
probed = []
|
|
|
|
async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs):
|
|
probed.extend(host["host_id"] for host in record["hosts"])
|
|
|
|
coordinator._reconcile_dataplane = capture_hosts
|
|
result = asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)))
|
|
|
|
assert probed == ["host-b"]
|
|
assert [host["host_id"] for host in result["hosts"]] == ["host-b"]
|
|
saved = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert [host["host_id"] for host in saved["hosts"]] == ["host-b"]
|
|
|
|
|
|
def test_new_revision_retires_host_without_exact_prior_removal_proof():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
first_payload = payload()
|
|
asyncio.run(coordinator.apply(first_payload))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
# A marker without the exact typed absence/readback fields is not proof.
|
|
durable["hosts"][0]["removed"] = True
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
|
|
|
|
probed = []
|
|
|
|
async def capture_hosts(plan, record, gate, previous_plan=None, **kwargs):
|
|
probed.extend(host["host_id"] for host in record["hosts"])
|
|
|
|
coordinator._reconcile_dataplane = capture_hosts
|
|
result = asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)))
|
|
|
|
assert probed == ["host-b", "host-a"]
|
|
second_durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
retiring = next(host for host in second_durable["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
assert retiring["retiring"] is True
|
|
assert retiring.get("removed") is not True
|
|
assert retiring["member_ids"] == ["member-a"]
|
|
assert retiring["retiring_members"] == [{
|
|
"member_id": "member-a",
|
|
"instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a",
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
|
|
}]
|
|
|
|
probed.clear()
|
|
third_payload = payload(
|
|
desired_revision=3,
|
|
operation_id="vpc-op-00000003",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)
|
|
third = asyncio.run(coordinator.apply(third_payload))
|
|
|
|
assert probed == ["host-b", "host-a"]
|
|
third_durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
carried = next(host for host in third_durable["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
assert carried["retiring"] is True
|
|
assert carried.get("removed") is not True
|
|
assert carried["member_ids"] == retiring["member_ids"]
|
|
assert carried["retiring_members"] == retiring["retiring_members"]
|
|
child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a"))
|
|
assert child["state"] == "deleting"
|
|
assert child.get("removed") is not True
|
|
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
host_a = next(host for host in durable["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
confirm_delete_hosts(parse_vpc_plan(third_payload), {"hosts": [host_a]})
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
|
|
coordinator._save_children(durable)
|
|
probed.clear()
|
|
fourth = asyncio.run(coordinator.apply(payload(
|
|
desired_revision=4,
|
|
operation_id="vpc-op-00000004",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)))
|
|
|
|
assert probed == ["host-b"]
|
|
assert [host["host_id"] for host in fourth["hosts"]] == ["host-b"]
|
|
child = json.loads(store.get("vpc_host", "vpc-hk-1.host-a"))
|
|
assert coordinator._terminal_removal(child)
|
|
assert child["desired_revision"] == 3
|
|
assert child["observed_revision"] == 3
|
|
|
|
root_delete_calls = []
|
|
|
|
async def confirm_root_delete(
|
|
plan, record, gate, previous_plan=None, **kwargs):
|
|
root_delete_calls.append(
|
|
[host["host_id"] for host in record["hosts"]])
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = confirm_root_delete
|
|
member_b = {
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"host_id": "host-b", "private_ip_cidr": "10.20.0.11/32",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
|
|
}
|
|
deleted = asyncio.run(coordinator.delete("vpc-hk-1", delete_payload(
|
|
desired_revision=5,
|
|
operation_id="vpc-delete-historical-proof-0001",
|
|
hosts=("host-a", "host-b"),
|
|
retiring_members=(retiring_member_a(), member_b),
|
|
)))
|
|
assert root_delete_calls == [["host-b"]]
|
|
assert deleted["removed_host_ids"] == ["host-a", "host-b"]
|
|
|
|
|
|
@pytest.mark.parametrize("corruption", ["not-list", "non-object", "missing"])
|
|
def test_new_revision_rejects_corrupt_durable_retiring_member_closure(corruption):
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
|
|
async def leave_pending(plan, record, gate, previous_plan=None, **kwargs):
|
|
return None
|
|
|
|
coordinator._reconcile_dataplane = leave_pending
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
host_a = next(host for host in durable["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
if corruption == "not-list":
|
|
host_a["retiring_members"] = {"member_id": "member-a"}
|
|
elif corruption == "non-object":
|
|
host_a["retiring_members"] = [None]
|
|
else:
|
|
host_a["retiring_members"] = []
|
|
tampered = json.dumps(durable)
|
|
store.set("vpc_desired", "vpc-hk-1", tampered)
|
|
|
|
with pytest.raises(VpcError) as rejected:
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=3,
|
|
operation_id="vpc-op-00000003",
|
|
hosts=[{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]}],
|
|
)))
|
|
assert rejected.value.error_code == "vpc_record_corrupt"
|
|
assert store.get("vpc_desired", "vpc-hk-1") == tampered
|
|
|
|
|
|
def test_apply_replay_retries_only_hosts_without_exact_removal_proof_after_restart():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": []},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
calls = []
|
|
|
|
async def remove_a_then_recover_b(
|
|
plan, record, gate, previous_plan=None, **kwargs):
|
|
host_ids = [host["host_id"] for host in record["hosts"]]
|
|
calls.append(host_ids)
|
|
if len(calls) == 1:
|
|
assert host_ids == ["host-a", "host-b"]
|
|
host_a = next(host for host in record["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
confirm_delete_hosts(plan, {"hosts": [host_a]})
|
|
host_b = next(host for host in record["hosts"]
|
|
if host["host_id"] == "host-b")
|
|
host_b.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": "vpc_host_unavailable",
|
|
})
|
|
return
|
|
# Host A is now offline. Reaching this branch with A in the execution
|
|
# closure would model the liveness regression this test prevents.
|
|
assert host_ids == ["host-b"]
|
|
record["hosts"][0].update({
|
|
"state": "active",
|
|
"removed": False,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"desired_revision": plan.desired_revision,
|
|
"observed_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_digest": plan.desired_digest,
|
|
"last_error_code": None,
|
|
})
|
|
|
|
coordinator._reconcile_dataplane = remove_a_then_recover_b
|
|
first = asyncio.run(coordinator.apply(target))
|
|
assert first["state"] != "active"
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
durable_hosts = {host["host_id"]: host for host in durable["hosts"]}
|
|
assert coordinator._confirmed_removal(
|
|
durable_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
|
|
assert durable_hosts["host-b"]["state"] == "uncertain"
|
|
|
|
coordinator.recover()
|
|
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]}
|
|
assert coordinator._confirmed_removal(
|
|
recovered_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
|
|
assert recovered_hosts["host-b"]["state"] == "uncertain"
|
|
|
|
result = asyncio.run(coordinator.apply(target))
|
|
assert calls == [["host-a", "host-b"], ["host-b"]]
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
final_hosts = {host["host_id"]: host for host in result["hosts"]}
|
|
assert coordinator._confirmed_removal(
|
|
final_hosts["host-a"], 1, parse_vpc_plan(target).desired_digest)
|
|
assert final_hosts["host-b"]["state"] == "active"
|
|
|
|
|
|
def test_reconcile_after_restart_retries_only_pending_normal_apply_hosts():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": []},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
|
|
async def partial(plan, record, gate, previous_plan=None, **kwargs):
|
|
host_a = next(host for host in record["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
confirm_delete_hosts(plan, {"hosts": [host_a]})
|
|
host_b = next(host for host in record["hosts"]
|
|
if host["host_id"] == "host-b")
|
|
host_b.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": "vpc_host_unavailable",
|
|
})
|
|
|
|
coordinator._reconcile_dataplane = partial
|
|
asyncio.run(coordinator.apply(target))
|
|
coordinator.recover()
|
|
|
|
restarted = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
calls = []
|
|
|
|
async def recover_b(plan, record, gate, previous_plan=None, **kwargs):
|
|
host_ids = [host["host_id"] for host in record["hosts"]]
|
|
calls.append(host_ids)
|
|
# Host A is intentionally unreachable after restart. It must not be
|
|
# part of the execution closure once its exact absence proof is
|
|
# durable; only the previously unavailable host B may be contacted.
|
|
assert host_ids == ["host-b"]
|
|
record["hosts"][0].update({
|
|
"state": "active",
|
|
"removed": False,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"desired_revision": plan.desired_revision,
|
|
"observed_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_digest": plan.desired_digest,
|
|
"last_error_code": None,
|
|
})
|
|
|
|
restarted._reconcile_dataplane_unlocked = recover_b
|
|
result = asyncio.run(restarted.reconcile("vpc-hk-1"))
|
|
|
|
assert calls == [["host-b"]]
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
final_hosts = {host["host_id"]: host for host in result["hosts"]}
|
|
plan = parse_vpc_plan(target)
|
|
assert restarted._confirmed_removal(
|
|
final_hosts["host-a"], plan.desired_revision, plan.desired_digest)
|
|
assert final_hosts["host-b"]["state"] == "active"
|
|
|
|
|
|
def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
first = asyncio.run(coordinator.apply(payload()))
|
|
digest = first["desired_digest"]
|
|
asyncio.run(coordinator.observe("vpc-hk-1", {
|
|
"host-a": {"state": "active", "observed_revision": 1,
|
|
"observed_digest": digest, "transport_verified": True,
|
|
"private_dataplane_verified": True},
|
|
}))
|
|
second = asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000002",
|
|
)))
|
|
assert second["state"] == "applying"
|
|
assert second["next_stage_allowed"] is False
|
|
assert second["observed_revision"] == 1
|
|
assert second["observed_digest"] == digest
|
|
assert second["hosts"][0]["observed_revision"] == 1
|
|
|
|
|
|
def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
|
|
async def fake_reconcile(plan, record, gate, previous_plan=None,
|
|
private_link_states=None, **_kwargs):
|
|
host = record["hosts"][0]
|
|
requested = (private_link_states or {}).get(INSTANCE_A, "preserve")
|
|
host.update(
|
|
state="active", observed_revision=plan.desired_revision,
|
|
observed_digest=plan.desired_digest, transport_verified=True,
|
|
private_dataplane_verified=True,
|
|
config_verified=True,
|
|
config_observed_revision=plan.desired_revision,
|
|
config_observed_digest=plan.desired_digest,
|
|
member_nics=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
|
|
"xml_verified": True,
|
|
"config_verified": True,
|
|
"seed_verified": True,
|
|
"restart_required": False,
|
|
"domain_running": True,
|
|
"guest_network_verified": requested != "down",
|
|
"guest_network_readback": (
|
|
"confirmed" if requested != "down" else "mismatch"),
|
|
"guest_reason_code": (
|
|
None if requested != "down"
|
|
else "vpc_private_guest_link_mismatch"),
|
|
"guest_private_mac": (derive_private_mac(
|
|
"vpc-hk-1", "member-a")
|
|
if requested != "down" else None),
|
|
"guest_private_ip_cidr": (
|
|
"10.20.0.10/32" if requested != "down" else None),
|
|
"guest_link_state": (
|
|
"up" if requested != "down" else None),
|
|
"guest_routes": [],
|
|
"link_state": "down" if requested == "down" else "up",
|
|
}],
|
|
)
|
|
|
|
coordinator._reconcile_dataplane = fake_reconcile
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a", link_state="down",
|
|
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
|
|
))
|
|
assert state["readback"] == "confirmed"
|
|
assert state["state"] == "suspended"
|
|
assert state["vpc_id"] == "vpc-hk-1"
|
|
assert state["member_id"] == "member-a"
|
|
assert state["link_state"] == "down"
|
|
assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a")
|
|
# Preserve does not implicitly request link-up from the coordinator.
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a", link_state="preserve",
|
|
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
|
|
))
|
|
assert state["link_state"] == "up"
|
|
|
|
|
|
def test_instance_reconcile_invalidates_stale_proof_when_fabric_is_inactive():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
|
|
async def confirm(plan, record, gate, previous_plan=None,
|
|
private_link_states=None, **_kwargs):
|
|
record["hosts"][0].update(
|
|
state="active", observed_revision=plan.desired_revision,
|
|
observed_digest=plan.desired_digest, transport_verified=True,
|
|
private_dataplane_verified=True, config_verified=True,
|
|
config_observed_revision=plan.desired_revision,
|
|
config_observed_digest=plan.desired_digest,
|
|
member_nics=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"interface": "private0",
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"private_mac": derive_private_mac(
|
|
"vpc-hk-1", "member-a"),
|
|
"xml_verified": True, "config_verified": True,
|
|
"seed_verified": True, "restart_required": False,
|
|
"domain_running": True, "guest_network_verified": True,
|
|
"guest_network_readback": "confirmed",
|
|
"guest_reason_code": None,
|
|
"guest_private_mac": derive_private_mac(
|
|
"vpc-hk-1", "member-a"),
|
|
"guest_private_ip_cidr": "10.20.0.10/32",
|
|
"guest_link_state": "up",
|
|
"guest_routes": [], "link_state": "up",
|
|
}],
|
|
)
|
|
|
|
coordinator._reconcile_dataplane = confirm
|
|
active = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a",
|
|
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
|
|
))
|
|
assert active["readback"] == "confirmed"
|
|
|
|
coordinator.fabric = FakeFabric(
|
|
state="degraded", gate=False, hosts=("host-a",))
|
|
coordinator._reconcile_dataplane = (
|
|
VpcCoordinator._reconcile_dataplane.__get__(
|
|
coordinator, VpcCoordinator))
|
|
stale = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a",
|
|
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
|
|
))
|
|
assert stale["state"] == "uncertain"
|
|
assert stale["readback"] == "missing"
|
|
|
|
|
|
def test_removed_member_reconcile_uses_durable_retiring_tombstone():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-a", "members": []}], members=[],
|
|
)))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A
|
|
|
|
async def fake_reconcile(plan, record, gate, previous_plan=None,
|
|
private_link_states=None, **_kwargs):
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = fake_reconcile
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a", link_state="absent",
|
|
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a",
|
|
"desired_state": "removed"},
|
|
))
|
|
assert state["state"] == "removed"
|
|
assert state["readback"] == "confirmed"
|
|
assert state["link_state"] == "absent"
|
|
|
|
|
|
def test_instance_reconcile_rejects_stale_revision_before_host_apply():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
calls = []
|
|
|
|
async def fake_reconcile(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
|
|
coordinator._reconcile_dataplane = fake_reconcile
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a",
|
|
expected={
|
|
"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a",
|
|
"desired_revision": 99,
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
|
|
},
|
|
))
|
|
assert calls == []
|
|
assert state["readback"] == "missing"
|
|
assert state["reason_code"] == "vpc_private_expectation_mismatch"
|
|
|
|
|
|
def test_instance_reconcile_rejects_host_mac_drift():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
|
|
async def fake_reconcile(plan, record, gate, previous_plan=None,
|
|
private_link_states=None, **_kwargs):
|
|
record["hosts"][0].update(
|
|
state="active", observed_revision=plan.desired_revision,
|
|
observed_digest=plan.desired_digest, transport_verified=True,
|
|
private_dataplane_verified=True,
|
|
config_verified=True,
|
|
config_observed_revision=plan.desired_revision,
|
|
config_observed_digest=plan.desired_digest,
|
|
member_nics=[{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
|
|
"private_mac": "02:00:00:00:00:01", "xml_verified": True,
|
|
"config_verified": True, "seed_verified": True,
|
|
"restart_required": False,
|
|
"domain_running": True,
|
|
"guest_network_verified": True,
|
|
"guest_network_readback": "confirmed",
|
|
"guest_reason_code": None,
|
|
"guest_private_mac": "02:00:00:00:00:01",
|
|
"guest_private_ip_cidr": "10.20.0.10/32",
|
|
"guest_link_state": "up",
|
|
"guest_routes": [],
|
|
"link_state": "up",
|
|
}],
|
|
)
|
|
|
|
coordinator._reconcile_dataplane = fake_reconcile
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a",
|
|
expected={"required": True, "vpc_id": "vpc-hk-1",
|
|
"member_id": "member-a",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
|
|
))
|
|
assert state["readback"] == "missing"
|
|
assert state["state"] == "uncertain"
|
|
|
|
|
|
def test_removed_member_reconcile_locks_retiring_vm_identity():
|
|
store = MemoryStore()
|
|
acquired = []
|
|
|
|
@asynccontextmanager
|
|
async def vm_lock(host_id, instance_uuid):
|
|
acquired.append((host_id, instance_uuid))
|
|
yield
|
|
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
|
|
asyncio.run(coordinator.apply(payload()))
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=2, operation_id="vpc-op-00000002",
|
|
hosts=[{"host_id": "host-a", "members": []}], members=[],
|
|
)))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
# One host pass removes every retiring member. Keep a sibling tombstone
|
|
# in the durable row to prove the lock set is not limited to the request
|
|
# target.
|
|
durable["hosts"][0]["retiring_members"].append({
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"host_id": "host-a", "private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
|
|
})
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
|
|
acquired.clear()
|
|
|
|
async def fake_unlocked(plan, record, gate, previous_plan=None,
|
|
private_link_states=None, **_kwargs):
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane_unlocked = fake_unlocked
|
|
state = asyncio.run(coordinator.reconcile_instance(
|
|
INSTANCE_A, host_id="host-a", link_state="absent",
|
|
expected={"required": True, "vpc_id": "vpc-hk-1",
|
|
"member_id": "member-a", "desired_state": "removed",
|
|
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
|
|
))
|
|
assert ("host-a", INSTANCE_A) in acquired
|
|
assert ("host-a", INSTANCE_B) in acquired
|
|
assert state["state"] == "removed"
|
|
assert state["readback"] == "confirmed"
|
|
|
|
|
|
def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation():
|
|
store = MemoryStore()
|
|
registry = SimpleNamespace(
|
|
components={"host-a": SimpleNamespace(store=store)},
|
|
)
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)), registry=registry,
|
|
)
|
|
key = f"host-scope.{INSTANCE_A}"
|
|
store.set("vm_rescue", key, json.dumps({
|
|
"vm_id": INSTANCE_A,
|
|
"operation_id": "rescue-op-1",
|
|
"status": "active",
|
|
}))
|
|
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-1") is None
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict"
|
|
|
|
store.set("vm_rescue", key, json.dumps({
|
|
"vm_id": INSTANCE_A,
|
|
"operation_id": "rescue-op-1",
|
|
"status": "staged",
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-1") is None
|
|
|
|
# A same-operation queued/failed row has not established a terminal
|
|
# rescue state and must remain a barrier.
|
|
for status in ("queued", "running", "exiting", "failed"):
|
|
store.set("vm_rescue", key, json.dumps({
|
|
"vm_id": INSTANCE_A,
|
|
"operation_id": "rescue-op-1",
|
|
"status": status,
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
|
|
|
|
|
|
def test_lifecycle_conflicts_follow_pending_host_execution_closure():
|
|
host_a_store = MemoryStore()
|
|
host_b_store = MemoryStore()
|
|
host_a_store.set("reinstall_job", "job-a", json.dumps({
|
|
"vm_id": INSTANCE_A,
|
|
"operation_id": "reinstall-op-a",
|
|
"status": "running",
|
|
}))
|
|
registry = SimpleNamespace(components={
|
|
"host-a": SimpleNamespace(store=host_a_store),
|
|
"host-b": SimpleNamespace(store=host_b_store),
|
|
})
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=registry,
|
|
)
|
|
plan = parse_vpc_plan(payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
]))
|
|
gate = {"active": True, "available": True}
|
|
full_record = coordinator._desired_record(
|
|
plan, gate, state="applying")
|
|
host_b = next(
|
|
host for host in full_record["hosts"]
|
|
if host["host_id"] == "host-b")
|
|
execution_record = dict(full_record)
|
|
execution_record["hosts"] = [dict(host_b)]
|
|
|
|
# Host A is not part of this pending-host retry, so its lifecycle job
|
|
# cannot starve host B's otherwise independent convergence pass.
|
|
assert coordinator._mark_lifecycle_conflicts(
|
|
plan, plan, execution_record) is False
|
|
assert execution_record["hosts"][0].get("last_error_code") is None
|
|
|
|
# A full revision mutation spans both hosts. One lifecycle conflict blocks
|
|
# and marks the whole execution closure before any host write begins.
|
|
assert coordinator._mark_lifecycle_conflicts(
|
|
plan, plan, full_record) is True
|
|
hosts = {host["host_id"]: host for host in full_record["hosts"]}
|
|
assert hosts["host-a"]["last_error_code"] == (
|
|
"vpc_vm_lifecycle_conflict")
|
|
assert hosts["host-b"]["last_error_code"] == (
|
|
"vpc_vm_lifecycle_conflict")
|
|
|
|
|
|
def test_lifecycle_conflicts_keep_same_host_sibling_barrier():
|
|
host_store = MemoryStore()
|
|
host_store.set("snapshot_job", "job-a", json.dumps({
|
|
"vm_id": INSTANCE_A,
|
|
"operation_id": "snapshot-op-a",
|
|
"status": "running",
|
|
}))
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-b",)),
|
|
registry=SimpleNamespace(components={
|
|
"host-b": SimpleNamespace(store=host_store),
|
|
}),
|
|
)
|
|
plan = parse_vpc_plan(payload(hosts=[{
|
|
"host_id": "host-b", "members": [
|
|
{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
},
|
|
{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
},
|
|
],
|
|
}]))
|
|
record = coordinator._desired_record(
|
|
plan, {"active": True, "available": True}, state="applying")
|
|
|
|
# A selected host pass may update shared bridge/WG/XML state for every
|
|
# local member. A sibling lifecycle job on that same host remains a hard
|
|
# barrier even when the originally pending member is different.
|
|
assert coordinator._mark_lifecycle_conflicts(
|
|
plan, plan, record) is True
|
|
assert record["hosts"][0]["last_error_code"] == (
|
|
"vpc_vm_lifecycle_conflict")
|
|
|
|
|
|
def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation():
|
|
store = MemoryStore()
|
|
registry = SimpleNamespace(
|
|
components={"host-a": SimpleNamespace(store=store)},
|
|
)
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)), registry=registry,
|
|
)
|
|
store.set("reinstall_job", "job-1", json.dumps({
|
|
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
|
|
"status": "uncertain",
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict"
|
|
|
|
store.set("reinstall_job", "job-1", json.dumps({
|
|
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
|
|
"status": "uncertain", "reinstall_committed": True,
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "reinstall-op-1") is None
|
|
|
|
store.set("reinstall_job", "job-1", json.dumps({
|
|
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
|
|
"status": "uncertain", "phase": "result_uncertain",
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "reinstall-op-1") is None
|
|
|
|
# Rescue job uncertainty is never exempted by the operation id alone;
|
|
# only the vm_rescue active/staged row proves its own terminal XML state.
|
|
store.delete("reinstall_job", "job-1")
|
|
store.set("rescue_job", "job-2", json.dumps({
|
|
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
|
|
"status": "uncertain",
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
|
|
store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({
|
|
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
|
|
"status": "active",
|
|
}))
|
|
assert coordinator._lifecycle_conflict(
|
|
"host-a", INSTANCE_A, "rescue-op-1") is None
|
|
|
|
|
|
@pytest.mark.parametrize("identity_change", [
|
|
{"region": "sg"},
|
|
{"fabric_id": "fabric-sg"},
|
|
{"fabric_revision": 8},
|
|
{"cidr": "10.21.0.0/24"},
|
|
])
|
|
def test_root_delete_rejects_catalog_identity_change_without_mutation(
|
|
identity_change):
|
|
store = MemoryStore()
|
|
fabric = FakeFabric(hosts=("host-a",))
|
|
coordinator = VpcCoordinator(store, fabric)
|
|
asyncio.run(coordinator.apply(payload()))
|
|
before = dict(store.values)
|
|
|
|
async def must_not_mutate(*_args, **_kwargs):
|
|
raise AssertionError("identity mismatch reached host mutation")
|
|
|
|
coordinator._reconcile_dataplane = must_not_mutate
|
|
fabric.get_status = lambda _fabric_id: (_ for _ in ()).throw(
|
|
AssertionError("identity mismatch reached Fabric gate"))
|
|
target = delete_payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-delete-identity-mismatch-0001",
|
|
**identity_change,
|
|
)
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == "vpc_delete_identity_mismatch"
|
|
# Compare the store's exact encoded values: a rejected delete must not
|
|
# rewrite timestamps, child projections, or the previous desired bytes.
|
|
assert store.values == before
|
|
|
|
|
|
def test_root_delete_without_previous_rejects_global_fabric_endpoint_overlap():
|
|
store = MemoryStore()
|
|
conflicting = parse_fabric_plan({
|
|
"fabric_id": "fabric-sg",
|
|
"region": "sg",
|
|
"overlay_cidr": "100.64.2.0/24",
|
|
"revision": 3,
|
|
"nodes": [
|
|
{"host_id": "host-c", "overlay_ip": "100.64.2.1",
|
|
"endpoint": "10.21.0.50:51820"},
|
|
{"host_id": "host-d", "overlay_ip": "100.64.2.2",
|
|
"endpoint": "203.0.113.4:51820"},
|
|
],
|
|
})
|
|
save_fabric_catalog(store, conflicting)
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
|
|
async def must_not_mutate(*_args, **_kwargs):
|
|
raise AssertionError("conflicting delete reached host mutation")
|
|
|
|
coordinator._reconcile_dataplane = must_not_mutate
|
|
before = dict(store.values)
|
|
target = delete_payload(
|
|
vpc_id="vpc-never-applied",
|
|
desired_revision=3,
|
|
operation_id="vpc-delete-catalog-overlap-0001",
|
|
cidr="10.21.0.0/24",
|
|
)
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(coordinator.delete("vpc-never-applied", target))
|
|
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == "vpc_fabric_endpoint_overlap"
|
|
assert store.values == before
|
|
|
|
|
|
def test_root_delete_without_previous_fails_closed_on_stale_fabric_digest():
|
|
store = MemoryStore()
|
|
fabric_plan = parse_fabric_plan({
|
|
"fabric_id": "fabric-sg",
|
|
"region": "sg",
|
|
"overlay_cidr": "100.64.2.0/24",
|
|
"revision": 3,
|
|
"nodes": [
|
|
{"host_id": "host-c", "overlay_ip": "100.64.2.1",
|
|
"endpoint": "203.0.113.3:51820"},
|
|
{"host_id": "host-d", "overlay_ip": "100.64.2.2",
|
|
"endpoint": "203.0.113.4:51820"},
|
|
],
|
|
})
|
|
save_fabric_catalog(store, fabric_plan, plan_digest="0" * 64)
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
before = dict(store.values)
|
|
target = delete_payload(
|
|
vpc_id="vpc-never-applied",
|
|
desired_revision=3,
|
|
operation_id="vpc-delete-stale-fabric-0001",
|
|
cidr="10.21.0.0/24",
|
|
)
|
|
|
|
with pytest.raises(VpcError) as error:
|
|
asyncio.run(coordinator.delete("vpc-never-applied", target))
|
|
|
|
assert error.value.status == 500
|
|
assert error.value.error_code == "vpc_fabric_record_corrupt"
|
|
assert store.values == before
|
|
|
|
|
|
def test_root_delete_accepts_revision_jump_and_superseded_unknown_host():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
calls = []
|
|
|
|
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
|
|
calls.append({
|
|
"host_ids": sorted(host["host_id"] for host in record["hosts"]),
|
|
"retiring": all(host.get("retiring") is True
|
|
for host in record["hosts"]),
|
|
"operation_id": kwargs.get("operation_id"),
|
|
})
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = confirm
|
|
target = delete_payload(
|
|
desired_revision=4,
|
|
operation_id="vpc-delete-revision-jump-0001",
|
|
hosts=("host-a", "host-b"),
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
|
|
assert result == {
|
|
"vpc_id": "vpc-hk-1",
|
|
"operation_id": "vpc-delete-revision-jump-0001",
|
|
"evidence_version": 2,
|
|
"desired_revision": 4,
|
|
"desired_digest": target["desired_digest"],
|
|
"state": "deleted",
|
|
"deleted": True,
|
|
"readback": "confirmed",
|
|
"removed_host_ids": ["host-a", "host-b"],
|
|
"completed_at": result["completed_at"],
|
|
}
|
|
assert result["completed_at"] > 0
|
|
assert calls == [{
|
|
"host_ids": ["host-a", "host-b"],
|
|
"retiring": True,
|
|
"operation_id": "vpc-delete-revision-jump-0001",
|
|
}]
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
assert store.list("vpc_host") == {}
|
|
durable_receipt = json.loads(store.get(
|
|
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
|
|
assert durable_receipt["readback"] == "confirmed"
|
|
assert durable_receipt["completed_at"] == result["completed_at"]
|
|
assert "retiring_members" not in durable_receipt
|
|
|
|
# A receipt written by an earlier engine build is compacted on replay;
|
|
# the digest retains the manifest commitment without permanently keeping
|
|
# member UUID/private IP/MAC identifiers after successful deletion.
|
|
legacy_receipt = dict(durable_receipt)
|
|
legacy_receipt["retiring_members"] = target["retiring_members"]
|
|
store.set(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1", json.dumps(legacy_receipt))
|
|
|
|
replay = asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert replay == result
|
|
compacted_raw = store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1")
|
|
compacted = json.loads(compacted_raw)
|
|
assert "retiring_members" not in compacted
|
|
assert INSTANCE_A not in compacted_raw
|
|
assert "10.20.0.10" not in compacted_raw
|
|
assert retiring_member_a()["private_mac"] not in compacted_raw
|
|
conflict = dict(target, operation_id="vpc-delete-revision-jump-0002")
|
|
with pytest.raises(VpcError) as changed:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", conflict))
|
|
assert changed.value.status == 409
|
|
assert changed.value.error_code == "vpc_delete_receipt_conflict"
|
|
with pytest.raises(VpcError) as recreated:
|
|
asyncio.run(coordinator.apply(payload(
|
|
desired_revision=5, operation_id="vpc-after-delete-0001")))
|
|
assert recreated.value.error_code == "vpc_deleted"
|
|
|
|
|
|
def test_delete_manifest_accepts_unicast_with_odd_last_octet_only():
|
|
derived = derive_private_mac("never", "m")
|
|
assert derived.endswith(":49")
|
|
valid = delete_payload(
|
|
vpc_id="never", cidr="10.24.0.0/24", desired_revision=1,
|
|
operation_id="vpc-delete-mac-boundary-0001", hosts=("host-a",),
|
|
retiring_members=({
|
|
"member_id": "m", "instance_uuid": INSTANCE_A,
|
|
"host_id": "host-a", "private_ip_cidr": "10.24.0.10/32",
|
|
"private_mac": derived,
|
|
},),
|
|
)
|
|
assert parse_vpc_delete_plan(valid)[1][0].private_mac == derived
|
|
invalid = dict(valid)
|
|
invalid.pop("desired_digest")
|
|
invalid["retiring_members"] = [{
|
|
**valid["retiring_members"][0],
|
|
"private_mac": "13:00:00:00:00:48",
|
|
}]
|
|
with pytest.raises(VpcError) as multicast:
|
|
parse_vpc_delete_plan(invalid)
|
|
assert multicast.value.error_code == "vpc_delete_manifest_invalid"
|
|
invalid["retiring_members"][0]["private_mac"] = "00:00:00:00:00:00"
|
|
with pytest.raises(VpcError) as all_zero:
|
|
parse_vpc_delete_plan(invalid)
|
|
assert all_zero.value.error_code == "vpc_delete_manifest_invalid"
|
|
|
|
|
|
def test_delete_digest_matches_shared_php_fixture():
|
|
fixture = json.loads((
|
|
Path(__file__).with_name("fixtures") / "vpc_delete_digest.json"
|
|
).read_text(encoding="utf-8"))
|
|
plan, retiring = parse_vpc_delete_plan(fixture["document"])
|
|
assert plan.desired_digest == fixture["expected_digest"]
|
|
assert plan.desired_digest == (
|
|
"2f83e2fe639c82702a21c200461cd4c59a57030ec7ba9603e949c5175de73314"
|
|
)
|
|
assert [host.host_id for host in plan.hosts] == ["11", "2"]
|
|
assert [member.member_id for member in retiring] == ["1" * 32]
|
|
|
|
|
|
def test_root_delete_recovers_required_host_after_restart():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
target = delete_payload(
|
|
operation_id="vpc-delete-recover-0001",
|
|
desired_revision=2,
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
|
|
async def interrupted(plan, record, gate, previous_plan=None, **kwargs):
|
|
# Simulate a process loss after retiring-member cleanup consumed its
|
|
# tombstone but before a host absence proof became durable.
|
|
for host in record["hosts"]:
|
|
host["retiring_members"] = []
|
|
|
|
coordinator._reconcile_dataplane = interrupted
|
|
with pytest.raises(VpcError) as pending:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert pending.value.status == 503
|
|
assert pending.value.error_code == "vpc_delete_incomplete"
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert durable["delete_target"]["required_host_ids"] == ["host-a"]
|
|
assert durable["hosts"][0]["retiring_members"] == []
|
|
|
|
coordinator.recover()
|
|
calls = []
|
|
|
|
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
|
|
calls.append({
|
|
"retiring": record["hosts"][0].get("retiring"),
|
|
"retiring_members": list(
|
|
record["hosts"][0].get("retiring_members", [])),
|
|
})
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = confirm
|
|
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
|
|
assert result["deleted"] is True
|
|
assert result["readback"] == "confirmed"
|
|
assert calls == [{
|
|
"retiring": True,
|
|
"retiring_members": [target["retiring_members"][0]],
|
|
}]
|
|
|
|
|
|
def test_root_delete_restart_retries_only_hosts_without_exact_removal_proof():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a", "host-b")))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
target = delete_payload(
|
|
operation_id="vpc-delete-partial-restart-0001",
|
|
desired_revision=2,
|
|
hosts=("host-a", "host-b"),
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
calls = []
|
|
|
|
async def partial_then_confirm(
|
|
plan, record, gate, previous_plan=None, **kwargs):
|
|
host_ids = [host["host_id"] for host in record["hosts"]]
|
|
calls.append(host_ids)
|
|
if len(calls) == 1:
|
|
assert host_ids == ["host-a", "host-b"]
|
|
host_a = next(
|
|
host for host in record["hosts"]
|
|
if host["host_id"] == "host-a")
|
|
confirm_delete_hosts(plan, {"hosts": [host_a]})
|
|
host_b = next(
|
|
host for host in record["hosts"]
|
|
if host["host_id"] == "host-b")
|
|
host_b.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": "vpc_host_unavailable",
|
|
})
|
|
return
|
|
assert host_ids == ["host-b"]
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = partial_then_confirm
|
|
with pytest.raises(VpcError) as incomplete:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert incomplete.value.error_code == "vpc_delete_incomplete"
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
durable_hosts = {host["host_id"]: host for host in durable["hosts"]}
|
|
assert coordinator._confirmed_removal(
|
|
durable_hosts["host-a"], 2, target["desired_digest"])
|
|
assert not coordinator._confirmed_removal(
|
|
durable_hosts["host-b"], 2, target["desired_digest"])
|
|
|
|
coordinator.recover()
|
|
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
recovered_hosts = {host["host_id"]: host for host in recovered["hosts"]}
|
|
assert coordinator._confirmed_removal(
|
|
recovered_hosts["host-a"], 2, target["desired_digest"])
|
|
assert recovered_hosts["host-b"]["state"] == "uncertain"
|
|
|
|
result = asyncio.run(coordinator.reconcile("vpc-hk-1"))
|
|
assert calls == [["host-a", "host-b"], ["host-b"]]
|
|
assert result["deleted"] is True
|
|
assert result["removed_host_ids"] == ["host-a", "host-b"]
|
|
assert result["readback"] == "confirmed"
|
|
|
|
|
|
def test_delete_pending_projection_does_not_probe_confirmed_offline_host():
|
|
target = delete_payload(
|
|
operation_id="vpc-delete-pending-probe-0001",
|
|
desired_revision=2,
|
|
hosts=("host-a", "host-b"),
|
|
)
|
|
plan, _manifest = parse_vpc_delete_plan(target)
|
|
nodes = (
|
|
FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
|
FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
|
)
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820, nodes,
|
|
"f" * 64,
|
|
)
|
|
probed = []
|
|
|
|
class RetryFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, retry_plan):
|
|
for node in retry_plan.nodes:
|
|
probed.append(node.host_id)
|
|
if node.host_id == "host-a":
|
|
raise FabricError(
|
|
"confirmed host is now offline", 502,
|
|
"fabric_host_capability_probe_failed",
|
|
)
|
|
return {
|
|
"host-b": (
|
|
SimpleNamespace(manager=object()),
|
|
{"host_id": "host-b"},
|
|
),
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), RetryFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
|
host_digests = []
|
|
host_peer_overlays = []
|
|
|
|
def confirm_host(*args, **kwargs):
|
|
host_digests.append(args[8])
|
|
host_peer_overlays.append(kwargs["fabric_peer_overlays"])
|
|
return {
|
|
"state": "active", "removed": True,
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"member_nics": [], "routes": [], "wg_allowed_ips": [],
|
|
"proxy_ips": [], "bridge": None, "bridge_owner": None,
|
|
"nft_table": None, "anti_spoof": False, "rp_filter": None,
|
|
"proxy_arp": None, "proxy_arp_pvlan": None,
|
|
"readback": "confirmed", "rollback": "none",
|
|
}
|
|
|
|
coordinator._run_host_dataplane = confirm_host
|
|
record = {
|
|
"vpc_id": plan.vpc_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"hosts": [{
|
|
"host_id": "host-b", "member_ids": [], "retiring": True,
|
|
"retiring_members": [],
|
|
"pending_member_mutations": [],
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
}],
|
|
}
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record,
|
|
{"active": False, "available": True},
|
|
previous_plan=plan,
|
|
operation_id=plan.operation_id,
|
|
))
|
|
|
|
assert probed == ["host-b"]
|
|
assert host_digests == [fabric_plan.digest]
|
|
assert host_peer_overlays == [{
|
|
"a" * 44: "100.64.0.1/32",
|
|
"b" * 44: "100.64.0.2/32",
|
|
}]
|
|
assert coordinator._confirmed_removal(
|
|
record["hosts"][0], plan.desired_revision, plan.desired_digest)
|
|
|
|
|
|
def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready():
|
|
target = payload(
|
|
desired_revision=2, operation_id="vpc-op-config-proof-0002",
|
|
hosts=[{"host_id": "host-a", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]}],
|
|
)
|
|
plan = parse_vpc_plan(target)
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
(FabricNode(
|
|
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),),
|
|
"f" * 64,
|
|
)
|
|
|
|
class RuntimeFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, _plan):
|
|
return {
|
|
"host-a": (
|
|
SimpleNamespace(manager=object()),
|
|
{"host_id": "host-a"},
|
|
),
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), RuntimeFabric(hosts=("host-a",)),
|
|
registry=object(),
|
|
)
|
|
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44},
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
|
coordinator._require_host_member_mutations_stopped = (
|
|
lambda *_args, **_kwargs: None)
|
|
coordinator._run_host_dataplane = lambda *args, **kwargs: {
|
|
"state": "applying", "transport_verified": True,
|
|
"private_dataplane_verified": False,
|
|
"config_verified": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"observed_revision": 0, "observed_digest": None,
|
|
"member_nics": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"config_verified": True, "xml_verified": True,
|
|
"domain_running": False, "guest_network_verified": False,
|
|
"restart_required": True,
|
|
}],
|
|
"reason_code": "vpc_private_guest_restart_required",
|
|
}
|
|
record = {
|
|
"vpc_id": plan.vpc_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"hosts": [{
|
|
"host_id": "host-a", "member_ids": ["member-b"],
|
|
"retiring_members": [retiring_member_a()],
|
|
"pending_member_mutations": [INSTANCE_A],
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
}],
|
|
}
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record, {"active": True, "available": True},
|
|
previous_plan=plan,
|
|
))
|
|
|
|
assert record["hosts"][0]["state"] == "applying"
|
|
assert record["hosts"][0]["retiring_members"] == []
|
|
|
|
|
|
def test_reconcile_skips_exact_host_so_later_host_gets_next_budget():
|
|
target = payload(
|
|
hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
],
|
|
)
|
|
plan = parse_vpc_plan(target)
|
|
gate = {"active": True, "available": True}
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
record = coordinator._desired_record(
|
|
plan, gate, state="applying")
|
|
calls = []
|
|
|
|
def exact(host):
|
|
member = next(
|
|
item for item in plan.members
|
|
if item.host_id == host["host_id"])
|
|
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
|
|
host.update({
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"observed_revision": plan.desired_revision,
|
|
"observed_digest": plan.desired_digest,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"retiring_members": [],
|
|
"pending_member_mutations": [],
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": private_mac,
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": True,
|
|
"guest_network_verified": True,
|
|
"restart_required": False,
|
|
}],
|
|
})
|
|
|
|
async def consume_one_host(_plan, execution, _gate, _previous, **kwargs):
|
|
host_ids = [host["host_id"] for host in execution["hosts"]]
|
|
calls.append(host_ids)
|
|
if len(calls) == 1:
|
|
exact(execution["hosts"][0])
|
|
execution["hosts"][1].update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"last_error_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
})
|
|
else:
|
|
assert host_ids == ["host-b"]
|
|
exact(execution["hosts"][0])
|
|
|
|
coordinator._reconcile_dataplane = consume_one_host
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
|
|
assert calls == [["host-a", "host-b"], ["host-b"]]
|
|
assert all(host["state"] == "active" for host in record["hosts"])
|
|
|
|
|
|
def test_reconcile_exact_active_hosts_return_without_periodic_probe():
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
plan = parse_vpc_plan(target)
|
|
gate = {"active": True, "available": True}
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
attempted_at = {"host-a": 200, "host-b": 100}
|
|
for host in record["hosts"]:
|
|
member = next(
|
|
item for item in plan.members
|
|
if item.host_id == host["host_id"])
|
|
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
|
|
host.update({
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"observed_revision": plan.desired_revision,
|
|
"observed_digest": plan.desired_digest,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"retiring_members": [],
|
|
"pending_member_mutations": [],
|
|
"probe_attempted_at_ns": attempted_at[host["host_id"]],
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": private_mac,
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": True,
|
|
"guest_network_verified": True,
|
|
"restart_required": False,
|
|
}],
|
|
})
|
|
|
|
calls = []
|
|
|
|
async def unexpected_probe(*args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
|
|
coordinator._reconcile_dataplane = unexpected_probe
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
|
|
assert calls == []
|
|
hosts = {host["host_id"]: host for host in record["hosts"]}
|
|
assert hosts["host-a"]["state"] == "active"
|
|
assert hosts["host-b"]["state"] == "active"
|
|
assert record["state"] == "active"
|
|
assert record["next_stage_allowed"] is True
|
|
assert record["private_dataplane_verified"] is True
|
|
|
|
|
|
def test_exact_stopped_configuration_does_not_reacquire_vm_locks():
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
plan = parse_vpc_plan(target)
|
|
gate = {"active": True, "available": True}
|
|
acquired = []
|
|
|
|
@asynccontextmanager
|
|
async def vm_lock(host_id, instance_uuid):
|
|
acquired.append((host_id, instance_uuid))
|
|
yield
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=object(), vm_lock=vm_lock,
|
|
)
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
for host in record["hosts"]:
|
|
member = next(
|
|
item for item in plan.members
|
|
if item.host_id == host["host_id"])
|
|
private_mac = derive_private_mac(plan.vpc_id, member.member_id)
|
|
host.update({
|
|
"host_id": member.host_id,
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"host_config_reusable": True,
|
|
"observed_revision": plan.desired_revision,
|
|
"observed_digest": plan.desired_digest,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"retiring_members": [],
|
|
"pending_member_mutations": [],
|
|
"probe_attempted_at_ns": 200 if member.host_id == "host-a" else 100,
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": private_mac,
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": True,
|
|
"guest_network_verified": True,
|
|
"restart_required": False,
|
|
}],
|
|
})
|
|
|
|
pending = next(
|
|
host for host in record["hosts"] if host["host_id"] == "host-b")
|
|
pending.update({
|
|
"state": "applying",
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
})
|
|
pending["member_nics"][0].update({
|
|
"domain_running": False,
|
|
"guest_network_verified": False,
|
|
"guest_network_readback": "stopped",
|
|
"guest_reason_code": "vpc_private_guest_stopped",
|
|
})
|
|
|
|
async def no_host_work(*_args, **_kwargs):
|
|
return None
|
|
|
|
coordinator._reconcile_dataplane_unlocked = no_host_work
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
assert acquired == []
|
|
|
|
acquired.clear()
|
|
asyncio.run(coordinator._reconcile_dataplane(
|
|
plan, record, gate, plan))
|
|
assert acquired == [
|
|
("host-a", INSTANCE_A),
|
|
("host-b", INSTANCE_B),
|
|
]
|
|
|
|
|
|
def test_instance_lock_wait_consumes_dataplane_deadline(monkeypatch):
|
|
plan = parse_vpc_plan(payload())
|
|
gate = {"active": True, "available": True}
|
|
observed_deadlines = []
|
|
|
|
@asynccontextmanager
|
|
async def slow_vm_lock(_host_id, _instance_uuid):
|
|
await asyncio.sleep(0.02)
|
|
yield
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a",)),
|
|
registry=object(), vm_lock=slow_vm_lock,
|
|
)
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
|
|
async def capture_deadline(*_args, request_deadline=None, **_kwargs):
|
|
observed_deadlines.append(request_deadline)
|
|
|
|
coordinator._reconcile_dataplane_unlocked = capture_deadline
|
|
monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.01)
|
|
asyncio.run(coordinator._reconcile_dataplane(
|
|
plan, record, gate, plan))
|
|
|
|
assert len(observed_deadlines) == 1
|
|
assert observed_deadlines[0] < time.monotonic()
|
|
|
|
|
|
def test_reconcile_rotates_pending_host_after_it_consumes_budget():
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
plan = parse_vpc_plan(target)
|
|
gate = {"active": True, "available": True}
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
record = coordinator._desired_record(
|
|
plan, gate, state="applying")
|
|
calls = []
|
|
|
|
async def consume_budget(_plan, execution, _gate, _previous, **kwargs):
|
|
calls.append([host["host_id"] for host in execution["hosts"]])
|
|
consumer = execution["hosts"][0]
|
|
consumer["probe_attempted_at_ns"] = 100 + len(calls)
|
|
consumer.update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"last_error_code":
|
|
"vpc_private_guest_probe_deadline_exceeded",
|
|
})
|
|
|
|
coordinator._reconcile_dataplane = consume_budget
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan))
|
|
|
|
assert calls == [["host-a", "host-b"], ["host-b", "host-a"]]
|
|
|
|
|
|
def test_instance_lifecycle_target_precedes_unrelated_pending_host():
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
plan = parse_vpc_plan(target)
|
|
gate = {"active": True, "available": True}
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
record["hosts"][1]["probe_attempted_at_ns"] = 999
|
|
calls = []
|
|
|
|
async def capture_order(_plan, execution, _gate, _previous, **kwargs):
|
|
calls.append([host["host_id"] for host in execution["hosts"]])
|
|
|
|
coordinator._reconcile_dataplane = capture_order
|
|
asyncio.run(coordinator._reconcile_pending_hosts(
|
|
plan, record, gate, plan,
|
|
private_link_states={INSTANCE_B: "down"},
|
|
))
|
|
|
|
assert calls == [["host-b", "host-a"]]
|
|
|
|
|
|
def _exact_guest_pending_host(plan):
|
|
member = plan.members[0]
|
|
return {
|
|
"host_id": member.host_id,
|
|
"member_ids": [member.member_id],
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"state": "applying",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": False,
|
|
"config_verified": True,
|
|
"host_config_reusable": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"retiring_members": [],
|
|
"pending_member_mutations": [],
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": derive_private_mac(
|
|
plan.vpc_id, member.member_id),
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": False,
|
|
"guest_network_verified": False,
|
|
"guest_network_readback": "stopped",
|
|
"guest_reason_code": "vpc_private_guest_stopped",
|
|
"restart_required": False,
|
|
}],
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(("live", "guest_state", "reason_code"), (
|
|
(None, "stopped", "vpc_private_guest_stopped"),
|
|
("running", "pending_restart", "vpc_private_guest_restart_required"),
|
|
("running", "unavailable", "vpc_private_guest_agent_unavailable"),
|
|
))
|
|
def test_guest_only_pending_host_uses_readback_lane_without_writes(
|
|
monkeypatch, live, guest_state, reason_code):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
plan = parse_vpc_plan(payload())
|
|
previous_host = _exact_guest_pending_host(plan)
|
|
events = []
|
|
monkeypatch.setattr(VMService, "__init__",
|
|
lambda self, *args, **kwargs: None)
|
|
|
|
def reject_ensure(*args, **kwargs):
|
|
raise AssertionError("runtime readback must not ensure private0")
|
|
|
|
monkeypatch.setattr(VMService, "ensure_private_interface", reject_ensure)
|
|
|
|
def read_pair(self, instance_uuid, **kwargs):
|
|
assert kwargs["private_seed_identity"] == {
|
|
"vpc_id": plan.vpc_id,
|
|
"member_id": plan.members[0].member_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"guest_network_generation":
|
|
kwargs["private_seed_identity"][
|
|
"guest_network_generation"],
|
|
}
|
|
assert len(kwargs["private_seed_identity"][
|
|
"guest_network_generation"]) == 64
|
|
events.append(("guest-read", instance_uuid, guest_state))
|
|
return {
|
|
"inactive": f"inactive:{instance_uuid}",
|
|
"live": (f"live:{instance_uuid}" if live else None),
|
|
"seed_verified": True,
|
|
"guest_network": {
|
|
"state": guest_state, "verified": False,
|
|
"reason_code": reason_code,
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "read_private_domain_pair", read_pair)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
|
|
def reject_apply(*args, **kwargs):
|
|
raise AssertionError("runtime readback must not apply host config")
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "apply", reject_apply)
|
|
|
|
def observe(self, dataplane, *, xml_by_instance=None):
|
|
events.append(("host-observe", tuple(xml_by_instance or {}),
|
|
xml_by_instance[INSTANCE_A]["guest_network"]["state"]))
|
|
pair = xml_by_instance[INSTANCE_A]
|
|
assert pair["seed_verified"] is True
|
|
assert pair["inactive"] == f"inactive:{INSTANCE_A}"
|
|
return {
|
|
"state": "applying",
|
|
"config_verified": True,
|
|
"host_config_reusable": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [],
|
|
"reason_code": reason_code,
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
result = coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
)
|
|
|
|
assert events == [
|
|
("guest-read", INSTANCE_A, guest_state),
|
|
("host-observe", (INSTANCE_A,), guest_state),
|
|
]
|
|
assert result["state"] == "applying"
|
|
|
|
|
|
def test_runtime_readback_gate_rejects_mutation_and_stale_config():
|
|
plan = parse_vpc_plan(payload())
|
|
host = _exact_guest_pending_host(plan)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
|
|
assert coordinator._runtime_readback_eligible(
|
|
host, plan, "host-a") is True
|
|
assert coordinator._runtime_readback_eligible(
|
|
host, plan, "host-a", {INSTANCE_A: "preserve"}) is True
|
|
assert coordinator._runtime_readback_eligible(
|
|
host, plan, "host-a", {INSTANCE_A: "down"}) is False
|
|
transport_retry = dict(host, transport_verified=False)
|
|
assert coordinator._runtime_readback_eligible(
|
|
transport_retry, plan, "host-a") is True
|
|
host_drift = dict(host, host_config_reusable=False)
|
|
assert coordinator._runtime_readback_eligible(
|
|
host_drift, plan, "host-a") is False
|
|
|
|
stale_revision = dict(host, config_observed_revision=0)
|
|
assert coordinator._runtime_readback_eligible(
|
|
stale_revision, plan, "host-a") is False
|
|
changed_members = dict(host, member_ids=[])
|
|
assert coordinator._runtime_readback_eligible(
|
|
changed_members, plan, "host-a") is False
|
|
retiring = dict(host, retiring_members=[{"member_id": "old-member"}])
|
|
assert coordinator._runtime_readback_eligible(
|
|
retiring, plan, "host-a") is False
|
|
assert coordinator._runtime_readback_eligible(
|
|
host, plan, "host-a", retiring_members=[{
|
|
"member_id": "old-member",
|
|
}]) is False
|
|
retiring_host = dict(host, retiring=True)
|
|
assert coordinator._runtime_readback_eligible(
|
|
retiring_host, plan, "host-a") is False
|
|
pending_mutation = dict(
|
|
host, pending_member_mutations=[INSTANCE_A])
|
|
assert coordinator._runtime_readback_eligible(
|
|
pending_mutation, plan, "host-a") is False
|
|
wrong_desired_digest = dict(host, desired_digest="0" * 64)
|
|
assert coordinator._runtime_readback_eligible(
|
|
wrong_desired_digest, plan, "host-a") is False
|
|
wrong_config_digest = dict(host, config_observed_digest="0" * 64)
|
|
assert coordinator._runtime_readback_eligible(
|
|
wrong_config_digest, plan, "host-a") is False
|
|
|
|
for field, value in (
|
|
("instance_uuid", INSTANCE_B),
|
|
("private_ip_cidr", "10.20.0.99/32"),
|
|
("private_mac", "02:00:00:00:00:01"),
|
|
("config_verified", False),
|
|
("xml_verified", False),
|
|
("seed_verified", False),
|
|
):
|
|
bad_nic = dict(host["member_nics"][0], **{field: value})
|
|
bad_host = dict(host, member_nics=[bad_nic])
|
|
assert coordinator._runtime_readback_eligible(
|
|
bad_host, plan, "host-a") is False
|
|
|
|
|
|
def test_runtime_readback_unstarted_probe_reports_deadline_not_missing_xml(
|
|
monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
plan = parse_vpc_plan(payload())
|
|
previous_host = _exact_guest_pending_host(plan)
|
|
monkeypatch.setattr(VMService, "__init__",
|
|
lambda self, *args, **kwargs: None)
|
|
monkeypatch.setattr(
|
|
VMService, "ensure_private_interface",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("deadline readback must not rewrite private0")),
|
|
)
|
|
monkeypatch.setattr(
|
|
VMService, "read_private_domain_pair",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("sub-second probe must not enter libvirt")),
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("deadline readback must not apply host config")),
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "observe",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("missing XML cannot be observed as topology")),
|
|
)
|
|
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
with pytest.raises(VpcError) as raised:
|
|
coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
request_deadline=time.monotonic() + 0.5,
|
|
)
|
|
|
|
assert raised.value.error_code == (
|
|
"vpc_private_guest_probe_deadline_exceeded")
|
|
|
|
|
|
def test_readback_topology_mismatch_revokes_config_proof(monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
plan = parse_vpc_plan(payload())
|
|
previous_host = _exact_guest_pending_host(plan)
|
|
previous_host["transport_verified"] = False
|
|
events = []
|
|
monkeypatch.setattr(VMService, "__init__",
|
|
lambda self, *args, **kwargs: None)
|
|
monkeypatch.setattr(
|
|
VMService, "ensure_private_interface",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("topology retry must not ensure private0")),
|
|
)
|
|
monkeypatch.setattr(
|
|
VMService, "read_private_domain_pair",
|
|
lambda self, instance_uuid, **kwargs: {
|
|
"inactive": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"seed_verified": True,
|
|
"guest_network": {
|
|
"state": "unavailable", "verified": False,
|
|
"reason_code": "vpc_private_guest_agent_unavailable",
|
|
},
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply",
|
|
lambda *args, **kwargs: (_ for _ in ()).throw(
|
|
AssertionError("topology retry must not apply host config")),
|
|
)
|
|
|
|
def observe(self, dataplane, *, xml_by_instance=None):
|
|
# This is the adapter boundary that runs the real observe script's
|
|
# bridge topology guard. Model a tap appearing after the old proof.
|
|
events.append("topology-observe")
|
|
return {
|
|
"state": "degraded",
|
|
"config_verified": False,
|
|
"host_config_reusable": False,
|
|
"config_observed_revision": 0,
|
|
"config_observed_digest": None,
|
|
"member_nics": [],
|
|
"reason_code": "vpc_private_dataplane_readback_mismatch",
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
observed = coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
)
|
|
|
|
assert events == ["topology-observe"]
|
|
assert observed["config_verified"] is False
|
|
assert observed["host_config_reusable"] is False
|
|
previous_host.update(observed)
|
|
assert coordinator._runtime_readback_eligible(
|
|
previous_host, plan, "host-a") is False
|
|
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply",
|
|
lambda self, dataplane: events.append(("apply", dataplane.host_id)),
|
|
)
|
|
|
|
def ensure(self, instance_uuid, **kwargs):
|
|
events.append(("ensure", instance_uuid))
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"seed_verified": True,
|
|
"restart_required": False,
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
|
|
coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
)
|
|
assert events == [
|
|
"topology-observe",
|
|
("apply", "host-a"),
|
|
("ensure", INSTANCE_A),
|
|
"topology-observe",
|
|
]
|
|
|
|
|
|
def test_readback_seed_mismatch_revokes_proof_then_retries_write(monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
plan = parse_vpc_plan(payload())
|
|
previous_host = _exact_guest_pending_host(plan)
|
|
events = []
|
|
writes_allowed = False
|
|
monkeypatch.setattr(VMService, "__init__",
|
|
lambda self, *args, **kwargs: None)
|
|
|
|
def apply(self, dataplane):
|
|
assert writes_allowed is True
|
|
events.append(("apply", dataplane.host_id))
|
|
|
|
def ensure(self, instance_uuid, **kwargs):
|
|
assert writes_allowed is True
|
|
events.append(("ensure", instance_uuid))
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}", "live": None,
|
|
"seed_verified": True, "restart_required": False,
|
|
}
|
|
|
|
def read_pair(self, instance_uuid, **kwargs):
|
|
assert writes_allowed is False
|
|
assert kwargs["private_seed_identity"]["desired_digest"] == (
|
|
plan.desired_digest)
|
|
events.append(("seed-read", instance_uuid))
|
|
return {
|
|
"inactive": f"inactive:{instance_uuid}", "live": None,
|
|
"seed_verified": False,
|
|
"guest_network": {
|
|
"state": "stopped", "verified": False,
|
|
"reason_code": "vpc_private_guest_stopped",
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
|
|
monkeypatch.setattr(VMService, "read_private_domain_pair", read_pair)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "apply", apply)
|
|
|
|
def observe(self, dataplane, *, xml_by_instance=None):
|
|
seed_verified = xml_by_instance[INSTANCE_A]["seed_verified"] is True
|
|
events.append(("observe", seed_verified))
|
|
return {
|
|
"state": "applying",
|
|
"config_verified": seed_verified,
|
|
"host_config_reusable": True,
|
|
"config_observed_revision": (
|
|
plan.desired_revision if seed_verified else 0),
|
|
"config_observed_digest": (
|
|
plan.desired_digest if seed_verified else None),
|
|
"member_nics": [{
|
|
**previous_host["member_nics"][0],
|
|
"seed_verified": seed_verified,
|
|
"config_verified": seed_verified,
|
|
"xml_verified": seed_verified,
|
|
}],
|
|
"reason_code": "vpc_private_guest_restart_required",
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
args = (
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan,
|
|
)
|
|
observed = coordinator._run_host_dataplane(
|
|
*args, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
)
|
|
|
|
assert events == [("seed-read", INSTANCE_A), ("observe", False)]
|
|
assert observed["config_verified"] is False
|
|
previous_host.update(observed)
|
|
assert coordinator._runtime_readback_eligible(
|
|
previous_host, plan, "host-a") is False
|
|
|
|
writes_allowed = True
|
|
coordinator._run_host_dataplane(
|
|
*args, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
previous_host=previous_host,
|
|
)
|
|
assert events == [
|
|
("seed-read", INSTANCE_A), ("observe", False),
|
|
("apply", "host-a"), ("ensure", INSTANCE_A), ("observe", True),
|
|
]
|
|
|
|
|
|
def test_explicit_private_link_change_keeps_full_write_lane(monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
plan = parse_vpc_plan(payload())
|
|
previous_host = _exact_guest_pending_host(plan)
|
|
events = []
|
|
monkeypatch.setattr(VMService, "__init__",
|
|
lambda self, *args, **kwargs: None)
|
|
|
|
def ensure(self, instance_uuid, **kwargs):
|
|
events.append(("ensure", instance_uuid, kwargs["link_state"]))
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}", "live": None,
|
|
"seed_verified": True, "restart_required": False,
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "ensure_private_interface", ensure)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply",
|
|
lambda self, dataplane: events.append(("apply", dataplane.host_id)),
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "observe",
|
|
lambda self, dataplane, **kwargs: {
|
|
"state": "applying",
|
|
"config_verified": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [],
|
|
"reason_code": "vpc_private_guest_stopped",
|
|
},
|
|
)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], plan, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
private_link_states={INSTANCE_A: "down"},
|
|
previous_host=previous_host,
|
|
)
|
|
|
|
assert events == [
|
|
("apply", "host-a"),
|
|
("ensure", INSTANCE_A, "down"),
|
|
]
|
|
|
|
|
|
def test_guest_probe_pool_is_rolling_bounded_and_deadline_fail_closed(
|
|
monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
members = [{
|
|
"member_id": f"member-{index}",
|
|
"instance_uuid":
|
|
f"00000000-0000-4000-8000-{index + 1:012x}",
|
|
"private_ip_cidr": f"10.20.0.{index + 10}/32",
|
|
"subnet_id": "subnet-a",
|
|
} for index in range(10)]
|
|
plan = parse_vpc_plan(payload(hosts=[{
|
|
"host_id": "host-a", "members": members,
|
|
}]))
|
|
monkeypatch.setattr("virsh_panel.vpc.VPC_GUEST_PROBE_WORKERS", 2)
|
|
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
|
|
|
|
def configure(self, instance_uuid, **kwargs):
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"restart_required": False,
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "ensure_private_interface", configure)
|
|
lock = threading.Lock()
|
|
active = 0
|
|
maximum = 0
|
|
started = []
|
|
|
|
def probe(self, instance_uuid, **kwargs):
|
|
nonlocal active, maximum
|
|
assert isinstance(kwargs.get("guest_probe_deadline"), float)
|
|
with lock:
|
|
active += 1
|
|
maximum = max(maximum, active)
|
|
started.append(instance_uuid)
|
|
time.sleep(0.1)
|
|
with lock:
|
|
active -= 1
|
|
return {
|
|
"inactive": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"guest_network": {
|
|
"state": "confirmed", "verified": True,
|
|
"source": "qemu_guest_agent",
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "read_private_domain_pair", probe)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply", lambda self, dataplane: {})
|
|
captured = {}
|
|
|
|
def observe(self, dataplane, *, xml_by_instance=None):
|
|
captured.update(xml_by_instance or {})
|
|
return {
|
|
"state": "applying",
|
|
"config_verified": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [],
|
|
"reason_code": "vpc_private_guest_probe_deadline_exceeded",
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
result = coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
request_deadline=time.monotonic() + 1.05,
|
|
)
|
|
|
|
assert maximum == 2
|
|
assert len(started) == 2
|
|
assert active == 0
|
|
assert len(captured) == 10
|
|
deadline_rows = [
|
|
pair["guest_network"] for pair in captured.values()
|
|
if pair["guest_network"].get("reason_code")
|
|
== "vpc_private_guest_probe_deadline_exceeded"
|
|
]
|
|
assert len(deadline_rows) == 8
|
|
assert result["config_verified"] is True
|
|
assert result["private_dataplane_verified"] is False
|
|
|
|
|
|
def test_guest_probe_default_pool_bounds_libvirt_reply_amplification():
|
|
from virsh_panel.vpc import VPC_GUEST_PROBE_WORKERS
|
|
|
|
# virDomainQemuAgentCommand materializes libvirt's bounded raw reply before
|
|
# Python can enforce its 192 KiB parse limit. Keep the default narrow even
|
|
# though individual tests may monkeypatch it lower for deterministic timing.
|
|
assert VPC_GUEST_PROBE_WORKERS == 4
|
|
|
|
|
|
def test_guest_probe_limit_is_process_wide_across_vpc_requests(monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
def make_plan(label, second_octet, first_uuid):
|
|
members = [{
|
|
"member_id": f"member-{label}-{index}",
|
|
"instance_uuid":
|
|
f"00000000-0000-4000-8000-{first_uuid + index:012x}",
|
|
"private_ip_cidr":
|
|
f"10.{second_octet}.0.{index + 10}/32",
|
|
"subnet_id": f"subnet-{label}",
|
|
} for index in range(4)]
|
|
return parse_vpc_plan(payload(
|
|
vpc_id=f"vpc-hk-{label}",
|
|
cidr=f"10.{second_octet}.0.0/24",
|
|
operation_id=f"vpc-op-{label}-00000001",
|
|
subnets=[{
|
|
"subnet_id": f"subnet-{label}",
|
|
"cidr": f"10.{second_octet}.0.0/25",
|
|
}],
|
|
hosts=[{"host_id": "host-a", "members": members}],
|
|
))
|
|
|
|
plans = (make_plan("a", 20, 1), make_plan("b", 21, 101))
|
|
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
|
|
monkeypatch.setattr(
|
|
VMService, "ensure_private_interface",
|
|
lambda self, instance_uuid, **kwargs: {
|
|
"xml": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"restart_required": False,
|
|
},
|
|
)
|
|
lock = threading.Lock()
|
|
release_full_wave = threading.Event()
|
|
active = 0
|
|
maximum = 0
|
|
started = 0
|
|
|
|
def probe(self, instance_uuid, **kwargs):
|
|
nonlocal active, maximum, started
|
|
with lock:
|
|
active += 1
|
|
started += 1
|
|
maximum = max(maximum, active)
|
|
if active >= 8:
|
|
release_full_wave.set()
|
|
# Without the process gate, both four-worker request pools enter
|
|
# together. With it, a bounded wave of four leaves before the next.
|
|
release_full_wave.wait(0.2)
|
|
with lock:
|
|
active -= 1
|
|
return {
|
|
"inactive": f"inactive:{instance_uuid}",
|
|
"live": f"live:{instance_uuid}",
|
|
"guest_network": {
|
|
"state": "confirmed", "verified": True,
|
|
"source": "qemu_guest_agent",
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(VMService, "read_private_domain_pair", probe)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "apply", lambda self, dataplane: {})
|
|
|
|
def observe(self, dataplane, *, xml_by_instance=None):
|
|
return {
|
|
"state": "active",
|
|
"config_verified": True,
|
|
"config_observed_revision": dataplane.vpc.desired_revision,
|
|
"config_observed_digest": dataplane.vpc.desired_digest,
|
|
"member_nics": [],
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe)
|
|
|
|
def run(plan):
|
|
return VpcCoordinator(
|
|
MemoryStore(), FakeFabric(),
|
|
)._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], None, fabric_public_keys={}, fabric_plan_digest="ef" * 32,
|
|
request_deadline=time.monotonic() + 3,
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as requests:
|
|
results = [future.result() for future in (
|
|
requests.submit(run, plans[0]),
|
|
requests.submit(run, plans[1]),
|
|
)]
|
|
|
|
assert started == 8
|
|
assert maximum == 4
|
|
assert all(result["config_verified"] is True for result in results)
|
|
|
|
|
|
@pytest.mark.parametrize("error_code", (
|
|
"vpc_member_requires_stopped",
|
|
"vpc_member_power_unavailable",
|
|
))
|
|
def test_member_stop_preflight_blocks_every_host_before_first_write(
|
|
error_code):
|
|
previous = parse_vpc_plan(payload())
|
|
target = payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-op-00000002",
|
|
hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
],
|
|
)
|
|
plan = parse_vpc_plan(target)
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
(
|
|
FabricNode("host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
|
FabricNode("host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
|
),
|
|
"f" * 64,
|
|
)
|
|
|
|
class PreflightFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, _plan):
|
|
return {
|
|
host_id: (SimpleNamespace(manager=object()), {"host_id": host_id})
|
|
for host_id in ("host-a", "host-b")
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), PreflightFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
|
checked = []
|
|
writes = []
|
|
|
|
def preflight(_profile, host_id, instance_uuids):
|
|
checked.append((host_id, tuple(instance_uuids)))
|
|
raise VpcError("member not safely stopped", 409, error_code)
|
|
|
|
coordinator._require_host_member_mutations_stopped = preflight
|
|
coordinator._run_host_dataplane = (
|
|
lambda *args, **kwargs: writes.append(args[1]) or {})
|
|
gate = {"active": True, "available": True}
|
|
record = coordinator._desired_record(
|
|
plan, gate, state="applying", previous_plan=previous)
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record, gate, previous_plan=previous))
|
|
|
|
assert checked == [("host-b", (INSTANCE_B,))]
|
|
assert writes == []
|
|
assert all(host["state"] != "uncertain" for host in record["hosts"])
|
|
blocked = next(host for host in record["hosts"]
|
|
if host["host_id"] == "host-b")
|
|
assert blocked["state"] == "applying"
|
|
assert blocked["last_error_code"] == error_code
|
|
assert blocked["pending_member_mutations"] == [INSTANCE_B]
|
|
|
|
|
|
def test_global_preflight_host_resolution_failure_stays_retryable():
|
|
plan = parse_vpc_plan(payload())
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
(FabricNode(
|
|
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),),
|
|
"f" * 64,
|
|
)
|
|
|
|
class UnavailableFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, _plan):
|
|
raise VpcError(
|
|
"host unavailable", 502, "vpc_host_unavailable")
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), UnavailableFabric(hosts=("host-a",)),
|
|
registry=object(),
|
|
)
|
|
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44},
|
|
}
|
|
writes = []
|
|
coordinator._run_host_dataplane = (
|
|
lambda *args, **kwargs: writes.append(args[1]) or {})
|
|
gate = {"active": True, "available": True}
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record, gate, previous_plan=None))
|
|
|
|
assert writes == []
|
|
assert record["state"] == "applying"
|
|
assert record["hosts"][0]["state"] == "applying"
|
|
assert record["hosts"][0]["last_error_code"] == "vpc_host_unavailable"
|
|
assert record["hosts"][0]["pending_member_mutations"] == [INSTANCE_A]
|
|
|
|
|
|
def test_stopped_member_config_proof_clears_mutation_and_periodic_work():
|
|
plan = parse_vpc_plan(payload())
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
(FabricNode(
|
|
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),),
|
|
"f" * 64,
|
|
)
|
|
|
|
class ConfigFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, _plan):
|
|
return {
|
|
"host-a": (
|
|
SimpleNamespace(manager=object()),
|
|
{"host_id": "host-a"},
|
|
),
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), ConfigFabric(hosts=("host-a",)), registry=object())
|
|
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44},
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
|
coordinator._require_host_member_mutations_stopped = (
|
|
lambda *_args, **_kwargs: None)
|
|
member = plan.members[0]
|
|
coordinator._run_host_dataplane = lambda *args, **kwargs: {
|
|
"state": "applying",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": False,
|
|
"config_verified": True,
|
|
"host_config_reusable": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": derive_private_mac(
|
|
plan.vpc_id, member.member_id),
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": False,
|
|
"guest_network_verified": False,
|
|
"guest_network_readback": "stopped",
|
|
"guest_reason_code": "vpc_private_guest_stopped",
|
|
"restart_required": False,
|
|
}],
|
|
"reason_code": "vpc_private_guest_stopped",
|
|
}
|
|
gate = {"active": True, "available": True}
|
|
record = coordinator._desired_record(plan, gate, state="applying")
|
|
assert record["hosts"][0]["pending_member_mutations"] == [INSTANCE_A]
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record, gate, previous_plan=None))
|
|
|
|
host = record["hosts"][0]
|
|
assert host["pending_member_mutations"] == []
|
|
assert host["state"] == "applying"
|
|
assert host["last_error_code"] == "vpc_private_guest_stopped"
|
|
assert coordinator._reconcile_host_selection(plan, record) == (set(), set())
|
|
coordinator._save("vpc_desired", plan.vpc_id, record)
|
|
|
|
@asynccontextmanager
|
|
async def unexpected_operation_scope(_plan):
|
|
raise AssertionError("periodic config-complete read must not claim")
|
|
yield
|
|
|
|
def unexpected_catalog_lock():
|
|
raise AssertionError("periodic config-complete read must not lock catalog")
|
|
|
|
coordinator._operation_scope = unexpected_operation_scope
|
|
coordinator._catalog_mutation_lock = unexpected_catalog_lock
|
|
periodic = asyncio.run(coordinator.reconcile(plan.vpc_id))
|
|
assert periodic["hosts"][0]["config_verified"] is True
|
|
assert periodic["hosts"][0]["private_dataplane_verified"] is False
|
|
|
|
|
|
def test_request_probe_deadline_stops_next_host_and_keeps_config_proof(
|
|
monkeypatch):
|
|
target = payload(hosts=[
|
|
{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a", "instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
{"host_id": "host-b", "members": [{
|
|
"member_id": "member-b", "instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}]},
|
|
])
|
|
plan = parse_vpc_plan(target)
|
|
fabric_plan = FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
(
|
|
FabricNode(
|
|
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
|
FabricNode(
|
|
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
|
),
|
|
"f" * 64,
|
|
)
|
|
|
|
class DeadlineFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, _plan):
|
|
return {
|
|
host_id: (
|
|
SimpleNamespace(manager=object()),
|
|
{"host_id": host_id},
|
|
)
|
|
for host_id in ("host-a", "host-b")
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
MemoryStore(), DeadlineFabric(hosts=("host-a", "host-b")),
|
|
registry=object(),
|
|
)
|
|
coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
|
coordinator._require_host_member_mutations_stopped = (
|
|
lambda *_args, **_kwargs: None)
|
|
monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.05)
|
|
calls = []
|
|
|
|
def slow_host(*args, **kwargs):
|
|
host_id = args[1]
|
|
calls.append(host_id)
|
|
time.sleep(0.08)
|
|
return {
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [{
|
|
"member_id": "member-a",
|
|
"guest_network_verified": True,
|
|
}],
|
|
"reason_code": None,
|
|
}
|
|
|
|
coordinator._run_host_dataplane = slow_host
|
|
gate = {"active": True, "available": True}
|
|
record = coordinator._desired_record(
|
|
plan, gate, state="applying")
|
|
|
|
asyncio.run(coordinator._reconcile_dataplane_unlocked(
|
|
plan, record, gate, previous_plan=plan))
|
|
|
|
assert calls == ["host-a"]
|
|
first, second = record["hosts"]
|
|
assert first["transport_verified"] is True
|
|
assert first["probe_attempted_at_ns"] > 0
|
|
assert first["config_verified"] is True
|
|
assert first["config_observed_revision"] == plan.desired_revision
|
|
assert first["private_dataplane_verified"] is False
|
|
assert first["member_nics"][0]["guest_network_verified"] is False
|
|
assert first["last_error_code"] \
|
|
== "vpc_private_guest_probe_deadline_exceeded"
|
|
assert second["transport_verified"] is False
|
|
assert second["last_error_code"] \
|
|
== "vpc_private_guest_probe_deadline_exceeded"
|
|
|
|
|
|
def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors():
|
|
deterministic_codes = (
|
|
"vpc_private_remove_requires_stopped",
|
|
"vpc_private_remove_unsafe",
|
|
"vpc_private_interface_conflict",
|
|
"vpc_seed_missing",
|
|
"vpc_seed_readback_invalid",
|
|
"vpc_seed_network_policy_migration_required",
|
|
"vpc_private_xml_invalid",
|
|
)
|
|
for index, error_code in enumerate(deterministic_codes, start=1):
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
target = delete_payload(
|
|
operation_id=f"vpc-delete-deterministic-{index:04d}",
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
|
|
async def blocked(plan, record, gate, previous_plan=None, **kwargs):
|
|
record["hosts"][0].update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": error_code,
|
|
})
|
|
|
|
coordinator._reconcile_dataplane = blocked
|
|
with pytest.raises(VpcError) as blocked_delete:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert blocked_delete.value.status == 409
|
|
assert blocked_delete.value.error_code == error_code
|
|
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
|
|
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
target = delete_payload(
|
|
operation_id="vpc-delete-transient-error-0001",
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
|
|
async def transient(plan, record, gate, previous_plan=None, **kwargs):
|
|
record["hosts"][0].update({
|
|
"state": "uncertain",
|
|
"transport_verified": False,
|
|
"private_dataplane_verified": False,
|
|
"observed_revision": 0,
|
|
"observed_digest": None,
|
|
"last_error_code": "vpc_private_remove_uncertain",
|
|
})
|
|
|
|
coordinator._reconcile_dataplane = transient
|
|
with pytest.raises(VpcError) as transient_delete:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert transient_delete.value.status == 503
|
|
assert transient_delete.value.error_code == "vpc_delete_incomplete"
|
|
|
|
|
|
@pytest.mark.parametrize("include_proven_host", [False, True])
|
|
def test_root_delete_carries_forward_exact_removed_host_without_recontact(
|
|
include_proven_host):
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
detached = payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-member-detach-0001",
|
|
subnets=[], members=[], routes=[], acls=[], hosts=[],
|
|
)
|
|
# Ordinary apply models the earlier member/deployment detach. Its host
|
|
# tombstone is then completed with the exact revision/digest proof that a
|
|
# lost PHP response may leave behind.
|
|
asyncio.run(coordinator.apply(detached))
|
|
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
assert durable["hosts"][0]["retiring"] is True
|
|
confirm_delete_hosts(parse_vpc_plan(detached), durable)
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
|
|
|
|
async def must_not_recontact(*args, **kwargs):
|
|
raise AssertionError("exact removed host must not be contacted again")
|
|
|
|
coordinator._reconcile_dataplane = must_not_recontact
|
|
target = delete_payload(
|
|
desired_revision=3,
|
|
operation_id=("vpc-delete-proven-included-0001"
|
|
if include_proven_host
|
|
else "vpc-delete-proven-omitted-0001"),
|
|
hosts=(("host-a",) if include_proven_host else ()),
|
|
)
|
|
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert result["removed_host_ids"] == ["host-a"]
|
|
assert result["readback"] == "confirmed"
|
|
|
|
|
|
def test_root_delete_requires_every_unproven_old_host_in_target():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
target = delete_payload(
|
|
desired_revision=2,
|
|
operation_id="vpc-delete-missing-host-0001",
|
|
hosts=(),
|
|
)
|
|
with pytest.raises(VpcError) as missing:
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert missing.value.status == 409
|
|
assert missing.value.error_code == "vpc_delete_host_set_mismatch"
|
|
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-hk-1") is None
|
|
|
|
|
|
def test_root_delete_without_desired_record_proves_requested_hosts_or_empty():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
calls = []
|
|
|
|
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
|
|
calls.append((
|
|
[host["host_id"] for host in record["hosts"]],
|
|
[host.get("retiring") for host in record["hosts"]],
|
|
[row["instance_uuid"] for host in record["hosts"]
|
|
for row in host.get("retiring_members", [])],
|
|
))
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = confirm
|
|
target = delete_payload(
|
|
vpc_id="vpc-never-applied",
|
|
operation_id="vpc-delete-never-applied-0001",
|
|
desired_revision=3,
|
|
hosts=("host-a",),
|
|
cidr="10.21.0.0/24",
|
|
retiring_members=({
|
|
"member_id": "member-never-applied",
|
|
"instance_uuid": INSTANCE_B,
|
|
"host_id": "host-a",
|
|
"private_ip_cidr": "10.21.0.10/32",
|
|
"private_mac": derive_private_mac(
|
|
"vpc-never-applied", "member-never-applied"),
|
|
},),
|
|
)
|
|
result = asyncio.run(coordinator.delete("vpc-never-applied", target))
|
|
assert result["removed_host_ids"] == ["host-a"]
|
|
assert calls == [(["host-a"], [True], [INSTANCE_B])]
|
|
|
|
empty = delete_payload(
|
|
vpc_id="vpc-empty-never-applied",
|
|
operation_id="vpc-delete-empty-never-applied-0001",
|
|
desired_revision=1,
|
|
hosts=(),
|
|
cidr="10.22.0.0/24",
|
|
)
|
|
result = asyncio.run(coordinator.delete(
|
|
"vpc-empty-never-applied", empty))
|
|
assert result["removed_host_ids"] == []
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_delete_manifest_removes_unknown_member_nic_before_host_resources(
|
|
monkeypatch):
|
|
import sys
|
|
import types
|
|
|
|
try:
|
|
import libvirt # noqa: F401
|
|
except ModuleNotFoundError:
|
|
stub = types.ModuleType("libvirt")
|
|
stub.libvirtError = type("libvirtError", (Exception,), {})
|
|
monkeypatch.setitem(sys.modules, "libvirt", stub)
|
|
from virsh_panel.libvirt_service import VMService
|
|
from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter
|
|
|
|
target = delete_payload(
|
|
vpc_id="vpc-remote-only",
|
|
operation_id="vpc-delete-remote-only-0001",
|
|
desired_revision=5,
|
|
hosts=("host-a",),
|
|
cidr="10.23.0.0/24",
|
|
retiring_members=({
|
|
"member_id": "member-remote-only",
|
|
"instance_uuid": INSTANCE_B,
|
|
"host_id": "host-a",
|
|
"private_ip_cidr": "10.23.0.10/32",
|
|
"private_mac": derive_private_mac(
|
|
"vpc-remote-only", "member-remote-only"),
|
|
},),
|
|
)
|
|
plan, manifest = parse_vpc_delete_plan(target)
|
|
rows = [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"host_id": member.host_id,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": member.private_mac,
|
|
} for member in manifest]
|
|
events = []
|
|
local_key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="
|
|
remote_key = "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI="
|
|
|
|
monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None)
|
|
|
|
def remove_nic(self, instance_uuid, **kwargs):
|
|
events.append(("nic", instance_uuid, kwargs["vpc_id"]))
|
|
return {"removed": True, "readback": "confirmed"}
|
|
|
|
monkeypatch.setattr(VMService, "remove_private_interface", remove_nic)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "__init__",
|
|
lambda self, manager, profile, **kwargs: None,
|
|
)
|
|
|
|
def remove_host(self, dataplane):
|
|
assert dataplane.fabric_peer_keys == frozenset({remote_key})
|
|
assert dataplane.fabric_peer_overlays == frozenset({
|
|
(remote_key, "100.64.0.3/32"),
|
|
})
|
|
events.append(("host", dataplane.host_id, dataplane.vpc.vpc_id))
|
|
return {
|
|
"state": "active", "removed": True,
|
|
"bridge": None, "bridge_owner": None, "nft_table": None,
|
|
"anti_spoof": False, "rp_filter": None,
|
|
"proxy_arp": None, "proxy_arp_pvlan": None,
|
|
"proxy_ips": [], "member_nics": [], "routes": [],
|
|
"wg_allowed_ips": [], "readback": "confirmed",
|
|
}
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "remove", remove_host)
|
|
coordinator = VpcCoordinator(MemoryStore(), FakeFabric())
|
|
observed = coordinator._run_host_dataplane(
|
|
plan, "host-a", SimpleNamespace(manager=object()),
|
|
{"libvirt_uri": "qemu:///system", "storage_pool": "default",
|
|
"iso_pool": "iso", "default_network": "default"},
|
|
[], None, retiring_members=rows,
|
|
fabric_public_keys={"host-a": local_key, "host-b": remote_key},
|
|
fabric_peer_overlays={
|
|
local_key: "100.64.0.2/32",
|
|
remote_key: "100.64.0.3/32",
|
|
},
|
|
fabric_plan_digest="ef" * 32,
|
|
)
|
|
assert events == [
|
|
("nic", INSTANCE_B, "vpc-remote-only"),
|
|
("host", "host-a", "vpc-remote-only"),
|
|
]
|
|
assert observed["removed"] is True
|
|
assert observed["readback"] == "confirmed"
|
|
|
|
|
|
def test_root_delete_receipt_survives_purge_failure_and_keeps_timestamp():
|
|
store = MemoryStore()
|
|
coordinator = VpcCoordinator(
|
|
store, FakeFabric(hosts=("host-a",)))
|
|
asyncio.run(coordinator.apply(payload()))
|
|
|
|
async def confirm(plan, record, gate, previous_plan=None, **kwargs):
|
|
confirm_delete_hosts(plan, record)
|
|
|
|
coordinator._reconcile_dataplane = confirm
|
|
target = delete_payload(
|
|
operation_id="vpc-delete-purge-crash-0001",
|
|
retiring_members=(retiring_member_a(),),
|
|
)
|
|
original_delete = store.delete
|
|
failed = False
|
|
|
|
def fail_once(kind, key):
|
|
nonlocal failed
|
|
if kind == "vpc_desired" and not failed:
|
|
failed = True
|
|
raise RuntimeError("crash after receipt commit")
|
|
return original_delete(kind, key)
|
|
|
|
store.delete = fail_once
|
|
with pytest.raises(RuntimeError, match="after receipt commit"):
|
|
asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
durable_receipt = json.loads(store.get(
|
|
VPC_DELETE_RECEIPT_KIND, "vpc-hk-1"))
|
|
first_completed_at = durable_receipt["completed_at"]
|
|
assert first_completed_at > 0
|
|
|
|
result = asyncio.run(coordinator.delete("vpc-hk-1", target))
|
|
assert result["completed_at"] == first_completed_at
|
|
assert result["readback"] == "confirmed"
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|