You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc.py

802 lines
32 KiB
Python

"""Structured VPC control-plane tests (no host data-plane commands)."""
from __future__ import annotations
import asyncio
import json
import threading
from contextlib import asynccontextmanager
from types import SimpleNamespace
import pytest
from virsh_panel.fabric import FabricError
from virsh_panel.vpc import (
VPC_SCHEMA_VERSION,
VpcCoordinator,
VpcError,
_plan_from_record,
parse_vpc_plan,
)
from virsh_panel.vpc_dataplane import derive_private_mac
INSTANCE_A = "11111111-1111-4111-8111-111111111111"
INSTANCE_B = "22222222-2222-4222-8222-222222222222"
class MemoryStore:
def __init__(self):
self.values = {}
self.lock = threading.RLock()
def get(self, kind, key):
with self.lock:
return self.values.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.values[kind, key] = value
def delete(self, kind, key):
with self.lock:
self.values.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.values.items()
if row_kind == kind}
class FakeFabric:
def __init__(self, *, state="active", gate=True, revision=7,
hosts=("host-a", "host-b")):
self.state = state
self.gate = gate
self.revision = revision
self.hosts = hosts
def get_status(self, fabric_id):
if fabric_id == "missing":
raise FabricError("missing", 404, "fabric_not_found")
return {
"fabric_id": fabric_id,
"region": "hk",
"revision": self.revision,
"state": self.state,
"next_stage_allowed": self.gate,
"nodes": [{"host_id": host} for host in self.hosts],
}
def payload(**overrides):
value = {
"schema_version": VPC_SCHEMA_VERSION,
"vpc_id": "vpc-hk-1",
"region": "hk",
"fabric_id": "fabric-hk",
"fabric_revision": 7,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-00000001",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
"routes": [],
"acls": [],
}
value.update(overrides)
return value
def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state():
first = parse_vpc_plan(payload())
second = parse_vpc_plan(payload(
hosts=None,
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.10",
"subnet_id": "subnet-a",
}],
))
assert first.desired_digest == second.desired_digest
assert first.members[0].private_ip == "10.20.0.10"
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="192.0.2.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="10.20.0.0/24",
network_cidr="10.21.0.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(subnets=[
{"subnet_id": "a", "cidr": "10.20.0.0/25"},
{"subnet_id": "b", "cidr": "10.20.0.64/26"},
]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "public", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "0.0.0.0/0",
"destination_cidr": "10.20.0.10/32", "priority": 1,
}]))
def test_member_uuid_matches_existing_libvirt_identity_boundary():
# Existing panel/libvirt records only require the canonical UUID shape;
# operator-supplied UUIDs may use non-RFC version/variant nibbles. The
# engine must accept those persisted VM identities in a VPC DTO too.
legacy_uuid = "11111111-1111-6111-0111-111111111111"
parsed = parse_vpc_plan(payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": legacy_uuid,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
assert parsed.members[0].instance_uuid == legacy_uuid
def test_digest_matches_php_wire_shape_for_host_member_grouping():
wire = payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
)
# PHP computes desired_digest before attaching operation_id to the
# signed/replayed payload; operation_id is an envelope field, not part of
# desired document identity.
digest_wire = {key: value for key, value in wire.items()
if key != "operation_id"}
expected = __import__("hashlib").sha256(
json.dumps(digest_wire, sort_keys=True, separators=(",", ":"),
ensure_ascii=True).encode("utf-8")
).hexdigest()
parsed = parse_vpc_plan(wire)
assert parsed.desired_digest == expected
assert parsed.canonical()["hosts"] == [{
"host_id": "host-a", "members": ["member-a"]
}]
def test_plan_validates_routes_acl_ports_and_member_scope():
parsed = parse_vpc_plan(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/32",
"next_hop_member_id": "member-a", "metric": 20}],
acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "port_from": 443,
"port_to": 443, "priority": 1}],
))
assert parsed.host_ids == {"host-a", "host-b"}
assert parsed.routes[0].next_hop_member_id == "member-a"
exact = parse_vpc_plan(payload(routes=[{
"route_id": "exact", "destination_cidr": "10.20.0.10/32",
"next_hop_member_id": "member-a",
}]))
assert exact.routes[0].destination_cidr == "10.20.0.10/32"
high_priority = parse_vpc_plan(payload(acls=[{
"acl_id": "high", "direction": "ingress", "action": "allow",
"protocol": "any", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "priority": 1_000_000,
}]))
assert high_priority.acls[0].priority == 1_000_000
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "bad", "destination_cidr": "0.0.0.0/0",
"next_hop_host_id": "host-a",
}]))
with pytest.raises(VpcError) as broad_route:
parse_vpc_plan(payload(routes=[{
"route_id": "broad", "destination_cidr": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
assert broad_route.value.error_code == "vpc_route_not_exact"
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "bad", "direction": "egress", "action": "allow",
"protocol": "icmp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.0/24", "port_from": 1,
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "alias-conflict",
"destination_cidr": "10.20.0.0/25",
"destination": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(hosts=[{
"host_id": "host-a", "members": ["member-a", "member-a"],
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(
hosts=[],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
def test_coordinator_persists_desired_children_and_blocks_without_fabric():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False))
result = asyncio.run(coordinator.apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["observed_revision"] == 0
assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False
assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"}
assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"}
assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"}
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert result["state"] == "applying"
assert result["next_stage_allowed"] is False
assert result["observed"]["fabric_gate"]["active"] is True
assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True
def test_recovery_reopens_only_the_control_plane_as_applying():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
asyncio.run(coordinator.apply(payload()))
coordinator.recover()
recovered = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert recovered["state"] == "applying"
assert recovered["next_stage_allowed"] is False
assert recovered["hosts"][0]["state"] == "applying"
assert recovered["hosts"][0]["transport_verified"] is True
def test_empty_vpc_cannot_open_fabric_gate():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
hosts=[], members=[],
)))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts"
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
assert result["state"] == "degraded"
assert result["fabric_gate"]["error_code"] == "vpc_no_members"
def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
)))
assert result["state"] == "applying"
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
digest = raw["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
assert observed["next_stage_allowed"] is True
assert observed["hosts"][0]["readback"] == {"safe": "ok"}
fabric.revision = 8
blocked = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert blocked["state"] == "degraded"
assert blocked["next_stage_allowed"] is False
def test_transport_only_observation_never_opens_private_dataplane_gate():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload()))
digest = result["desired_digest"]
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
}))
assert pending["state"] == "applying"
assert pending["next_stage_allowed"] is False
assert pending["private_dataplane_verified"] is False
assert pending["hosts"][0]["state"] == "applying"
assert pending["hosts"][0]["last_error_code"] == (
"vpc_private_dataplane_unverified"
)
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
)))
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert observed["state"] == "degraded"
assert observed["next_stage_allowed"] is False
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": False,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert inconsistent.value.error_code == "vpc_observation_invalid"
def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def wrong_region(fabric_id):
status = original(fabric_id)
status["region"] = "us"
return status
fabric.get_status = wrong_region
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_fabric_region_mismatch"
@pytest.mark.parametrize("mutation", [
lambda status: status.update(fabric_id="other-fabric"),
lambda status: status.update(revision=True),
])
def test_fabric_gate_rejects_tampered_identity_or_revision(mutation):
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def tampered(fabric_id):
status = original(fabric_id)
mutation(status)
return status
fabric.get_status = tampered
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["next_stage_allowed"] is False
assert result["state"] == "degraded"
def test_coordinator_revision_conflict_and_tampered_record_fail_closed():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
first = asyncio.run(coordinator.apply(payload()))
replay = asyncio.run(coordinator.apply(payload()))
assert replay["desired_digest"] == first["desired_digest"]
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.apply(payload(
operation_id="vpc-op-00000002",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert error.value.error_code == "vpc_revision_conflict"
with pytest.raises(VpcError) as invalid_revision:
asyncio.run(coordinator.apply(payload(
desired_revision=0, operation_id="vpc-op-00000003",
)))
assert invalid_revision.value.error_code == "vpc_invalid"
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)))
with pytest.raises(VpcError) as stale:
asyncio.run(coordinator.apply(payload(
desired_revision=1, operation_id="vpc-op-00000005",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert stale.value.error_code == "vpc_revision_conflict"
tampered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
tampered["desired_digest"] = "0" * 64
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as corrupt:
asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert corrupt.value.error_code == "vpc_record_corrupt"
tampered["vpc_id"] = "other-vpc"
tampered["desired_digest"] = first["desired_digest"]
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as mismatched:
coordinator.get_status("vpc-hk-1")
assert mismatched.value.error_code == "vpc_record_corrupt"
# Restore a structurally valid root and corrupt only observed evidence;
# a newer target must not overwrite that evidence silently.
restored = json.loads(store.get("vpc_desired", "vpc-hk-1"))
restored["vpc_id"] = "vpc-hk-1"
restored["desired_revision"] = 2
restored["desired_digest"] = parse_vpc_plan(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)).desired_digest
restored["observed_revision"] = 99
store.set("vpc_desired", "vpc-hk-1", json.dumps(restored))
with pytest.raises(VpcError) as observed_corrupt:
asyncio.run(coordinator.apply(payload(
desired_revision=3, operation_id="vpc-op-00000006",
)))
assert observed_corrupt.value.error_code == "vpc_record_corrupt"
def test_recover_invalidates_observed_gate_and_record_round_trips():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
raw["state"] = "active"
raw["next_stage_allowed"] = True
raw["observed_revision"] = 1
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
transport_verified=True,
private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert recovered["state"] == "uncertain"
assert recovered["next_stage_allowed"] is False
assert recovered["observed_revision"] == 1
assert recovered["observed_digest"] == recovered["desired_digest"]
assert recovered["fabric_gate"]["active"] is False
assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending"
assert recovered["hosts"][0]["observed_revision"] == 1
assert _plan_from_record(recovered).vpc_id == "vpc-hk-1"
def test_removed_children_are_tombstoned_until_observed_cleanup():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
changed = payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}],
members=[],
)
asyncio.run(coordinator.apply(changed))
old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a"))
assert old["state"] == "removed"
assert old["removed"] is True
def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
first = asyncio.run(coordinator.apply(payload()))
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
)))
assert second["state"] == "applying"
assert second["next_stage_allowed"] is False
assert second["observed_revision"] == 1
assert second["observed_digest"] == digest
assert second["hosts"][0]["observed_revision"] == 1
def test_instance_reconcile_returns_host_verified_private_state_and_preserves_link():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
requested = (private_link_states or {}).get(INSTANCE_A, "preserve")
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
"xml_verified": True,
"link_state": "down" if requested == "down" else "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="down",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["readback"] == "confirmed"
assert state["state"] == "suspended"
assert state["vpc_id"] == "vpc-hk-1"
assert state["member_id"] == "member-a"
assert state["link_state"] == "down"
assert state["private_mac"] == derive_private_mac("vpc-hk-1", "member-a")
# Preserve does not implicitly request link-up from the coordinator.
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="preserve",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"},
))
assert state["link_state"] == "up"
def test_removed_member_reconcile_uses_durable_retiring_tombstone():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert durable["hosts"][0]["retiring_members"][0]["instance_uuid"] == INSTANCE_A
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True, member_nics=[],
retiring_members=[],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_state": "removed"},
))
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
assert state["link_state"] == "absent"
def test_instance_reconcile_rejects_stale_revision_before_host_apply():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
calls = []
async def fake_reconcile(*args, **kwargs):
calls.append((args, kwargs))
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={
"required": True, "vpc_id": "vpc-hk-1", "member_id": "member-a",
"desired_revision": 99,
"private_mac": derive_private_mac("vpc-hk-1", "member-a"),
},
))
assert calls == []
assert state["readback"] == "missing"
assert state["reason_code"] == "vpc_private_expectation_mismatch"
def test_instance_reconcile_rejects_host_mac_drift():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
async def fake_reconcile(plan, record, gate, previous_plan=None,
private_link_states=None):
record["hosts"][0].update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True,
member_nics=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"interface": "private0", "private_ip_cidr": "10.20.0.10/32",
"private_mac": "02:00:00:00:00:01", "xml_verified": True,
"link_state": "up",
}],
)
coordinator._reconcile_dataplane = fake_reconcile
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert state["readback"] == "missing"
assert state["state"] == "uncertain"
def test_removed_member_reconcile_locks_retiring_vm_identity():
store = MemoryStore()
acquired = []
@asynccontextmanager
async def vm_lock(host_id, instance_uuid):
acquired.append((host_id, instance_uuid))
yield
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), vm_lock=vm_lock)
asyncio.run(coordinator.apply(payload()))
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
durable = json.loads(store.get("vpc_desired", "vpc-hk-1"))
# One host pass removes every retiring member. Keep a sibling tombstone
# in the durable row to prove the lock set is not limited to the request
# target.
durable["hosts"][0]["retiring_members"].append({
"member_id": "member-b", "instance_uuid": INSTANCE_B,
"host_id": "host-a", "private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
"private_mac": derive_private_mac("vpc-hk-1", "member-b"),
})
store.set("vpc_desired", "vpc-hk-1", json.dumps(durable))
acquired.clear()
async def fake_unlocked(plan, record, gate, previous_plan=None,
private_link_states=None):
host = record["hosts"][0]
host.update(
state="active", observed_revision=plan.desired_revision,
observed_digest=plan.desired_digest, transport_verified=True,
private_dataplane_verified=True, member_nics=[],
retiring_members=[],
)
coordinator._reconcile_dataplane_unlocked = fake_unlocked
state = asyncio.run(coordinator.reconcile_instance(
INSTANCE_A, host_id="host-a", link_state="absent",
expected={"required": True, "vpc_id": "vpc-hk-1",
"member_id": "member-a", "desired_state": "removed",
"private_mac": derive_private_mac("vpc-hk-1", "member-a")},
))
assert ("host-a", INSTANCE_A) in acquired
assert ("host-a", INSTANCE_B) in acquired
assert state["state"] == "removed"
assert state["readback"] == "confirmed"
def test_lifecycle_conflict_allows_only_same_terminal_rescue_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
key = f"host-scope.{INSTANCE_A}"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-2") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": "staged",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None
# A same-operation queued/failed row has not established a terminal
# rescue state and must remain a barrier.
for status in ("queued", "running", "exiting", "failed"):
store.set("vm_rescue", key, json.dumps({
"vm_id": INSTANCE_A,
"operation_id": "rescue-op-1",
"status": status,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
def test_lifecycle_conflict_requires_reinstall_commit_for_same_operation():
store = MemoryStore()
registry = SimpleNamespace(
components={"host-a": SimpleNamespace(store=store)},
)
coordinator = VpcCoordinator(
store, FakeFabric(hosts=("host-a",)), registry=registry,
)
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") == "vpc_vm_lifecycle_conflict"
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "reinstall_committed": True,
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
store.set("reinstall_job", "job-1", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "reinstall-op-1",
"status": "uncertain", "phase": "result_uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "reinstall-op-1") is None
# Rescue job uncertainty is never exempted by the operation id alone;
# only the vm_rescue active/staged row proves its own terminal XML state.
store.delete("reinstall_job", "job-1")
store.set("rescue_job", "job-2", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "uncertain",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") == "vpc_vm_lifecycle_conflict"
store.set("vm_rescue", f"host-scope.{INSTANCE_A}", json.dumps({
"vm_id": INSTANCE_A, "operation_id": "rescue-op-1",
"status": "active",
}))
assert coordinator._lifecycle_conflict(
"host-a", INSTANCE_A, "rescue-op-1") is None