You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc.py

529 lines
21 KiB
Python

"""Structured VPC control-plane tests (no host data-plane commands)."""
from __future__ import annotations
import asyncio
import json
import threading
import pytest
from virsh_panel.fabric import FabricError
from virsh_panel.vpc import (
VPC_SCHEMA_VERSION,
VpcCoordinator,
VpcError,
_plan_from_record,
parse_vpc_plan,
)
INSTANCE_A = "11111111-1111-4111-8111-111111111111"
INSTANCE_B = "22222222-2222-4222-8222-222222222222"
class MemoryStore:
def __init__(self):
self.values = {}
self.lock = threading.RLock()
def get(self, kind, key):
with self.lock:
return self.values.get((kind, key))
def set(self, kind, key, value):
with self.lock:
self.values[kind, key] = value
def delete(self, kind, key):
with self.lock:
self.values.pop((kind, key), None)
def list(self, kind):
with self.lock:
return {key: value for (row_kind, key), value in self.values.items()
if row_kind == kind}
class FakeFabric:
def __init__(self, *, state="active", gate=True, revision=7,
hosts=("host-a", "host-b")):
self.state = state
self.gate = gate
self.revision = revision
self.hosts = hosts
def get_status(self, fabric_id):
if fabric_id == "missing":
raise FabricError("missing", 404, "fabric_not_found")
return {
"fabric_id": fabric_id,
"region": "hk",
"revision": self.revision,
"state": self.state,
"next_stage_allowed": self.gate,
"nodes": [{"host_id": host} for host in self.hosts],
}
def payload(**overrides):
value = {
"schema_version": VPC_SCHEMA_VERSION,
"vpc_id": "vpc-hk-1",
"region": "hk",
"fabric_id": "fabric-hk",
"fabric_revision": 7,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-00000001",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
"routes": [],
"acls": [],
}
value.update(overrides)
return value
def test_plan_canonicalizes_members_and_rejects_public_or_overlapping_state():
first = parse_vpc_plan(payload())
second = parse_vpc_plan(payload(
hosts=None,
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.10",
"subnet_id": "subnet-a",
}],
))
assert first.desired_digest == second.desired_digest
assert first.members[0].private_ip == "10.20.0.10"
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="192.0.2.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(cidr="10.20.0.0/24",
network_cidr="10.21.0.0/24"))
with pytest.raises(VpcError):
parse_vpc_plan(payload(subnets=[
{"subnet_id": "a", "cidr": "10.20.0.0/25"},
{"subnet_id": "b", "cidr": "10.20.0.64/26"},
]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "public", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "0.0.0.0/0",
"destination_cidr": "10.20.0.10/32", "priority": 1,
}]))
def test_member_uuid_matches_existing_libvirt_identity_boundary():
# Existing panel/libvirt records only require the canonical UUID shape;
# operator-supplied UUIDs may use non-RFC version/variant nibbles. The
# engine must accept those persisted VM identities in a VPC DTO too.
legacy_uuid = "11111111-1111-6111-0111-111111111111"
parsed = parse_vpc_plan(payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": legacy_uuid,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
assert parsed.members[0].instance_uuid == legacy_uuid
def test_digest_matches_php_wire_shape_for_host_member_grouping():
wire = payload(
hosts=[{"host_id": "host-a", "members": ["member-a"]}],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
)
# PHP computes desired_digest before attaching operation_id to the
# signed/replayed payload; operation_id is an envelope field, not part of
# desired document identity.
digest_wire = {key: value for key, value in wire.items()
if key != "operation_id"}
expected = __import__("hashlib").sha256(
json.dumps(digest_wire, sort_keys=True, separators=(",", ":"),
ensure_ascii=True).encode("utf-8")
).hexdigest()
parsed = parse_vpc_plan(wire)
assert parsed.desired_digest == expected
assert parsed.canonical()["hosts"] == [{
"host_id": "host-a", "members": ["member-a"]
}]
def test_plan_validates_routes_acl_ports_and_member_scope():
parsed = parse_vpc_plan(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
routes=[{"route_id": "route-a", "destination_cidr": "10.20.0.128/25",
"next_hop_member_id": "member-a", "metric": 20}],
acls=[{"acl_id": "acl-a", "direction": "ingress", "action": "allow",
"protocol": "tcp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "port_from": 443,
"port_to": 443, "priority": 1}],
))
assert parsed.host_ids == {"host-a", "host-b"}
assert parsed.routes[0].next_hop_member_id == "member-a"
exact = parse_vpc_plan(payload(routes=[{
"route_id": "exact", "destination_cidr": "10.20.0.10/32",
"next_hop_member_id": "member-a",
}]))
assert exact.routes[0].destination_cidr == "10.20.0.10/32"
high_priority = parse_vpc_plan(payload(acls=[{
"acl_id": "high", "direction": "ingress", "action": "allow",
"protocol": "any", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.10/32", "priority": 1_000_000,
}]))
assert high_priority.acls[0].priority == 1_000_000
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "bad", "destination_cidr": "0.0.0.0/0",
"next_hop_host_id": "host-a",
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(acls=[{
"acl_id": "bad", "direction": "egress", "action": "allow",
"protocol": "icmp", "source_cidr": "10.20.0.0/24",
"destination_cidr": "10.20.0.0/24", "port_from": 1,
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(routes=[{
"route_id": "alias-conflict",
"destination_cidr": "10.20.0.0/25",
"destination": "10.20.0.128/25",
"next_hop_member_id": "member-a",
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(hosts=[{
"host_id": "host-a", "members": ["member-a", "member-a"],
}]))
with pytest.raises(VpcError):
parse_vpc_plan(payload(
hosts=[],
members=[{
"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
))
def test_coordinator_persists_desired_children_and_blocks_without_fabric():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(state="degraded", gate=False))
result = asyncio.run(coordinator.apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["observed_revision"] == 0
assert result["observed"]["hosts"]["host-a"]["next_stage_allowed"] is False
assert set(store.list("vpc_subnet")) == {"vpc-hk-1.subnet-a"}
assert set(store.list("vpc_member")) == {"vpc-hk-1.member-a"}
assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"}
def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload()))
assert result["state"] == "applying"
assert result["next_stage_allowed"] is False
assert result["observed"]["fabric_gate"]["active"] is True
assert result["observed"]["fabric_gate"]["next_stage_allowed"] is True
def test_recovery_reopens_only_the_control_plane_as_applying():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
asyncio.run(coordinator.apply(payload()))
coordinator.recover()
recovered = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert recovered["state"] == "applying"
assert recovered["next_stage_allowed"] is False
assert recovered["hosts"][0]["state"] == "applying"
assert recovered["hosts"][0]["transport_verified"] is True
def test_empty_vpc_cannot_open_fabric_gate():
store = MemoryStore()
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
hosts=[], members=[],
)))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_no_participating_hosts"
result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}], members=[],
)))
assert result["state"] == "degraded"
assert result["fabric_gate"]["error_code"] == "vpc_no_members"
def test_coordinator_requires_exact_fabric_revision_and_all_host_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]}],
)))
assert result["state"] == "applying"
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
digest = raw["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True,
"readback": {"safe": "ok", "oversized": "x" * 513}},
}))
assert observed["state"] == "active"
assert observed["next_stage_allowed"] is True
assert observed["hosts"][0]["readback"] == {"safe": "ok"}
fabric.revision = 8
blocked = asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert blocked["state"] == "degraded"
assert blocked["next_stage_allowed"] is False
def test_transport_only_observation_never_opens_private_dataplane_gate():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload()))
digest = result["desired_digest"]
pending = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True},
}))
assert pending["state"] == "applying"
assert pending["next_stage_allowed"] is False
assert pending["private_dataplane_verified"] is False
assert pending["hosts"][0]["state"] == "applying"
assert pending["hosts"][0]["last_error_code"] == (
"vpc_private_dataplane_unverified"
)
def test_partial_host_observation_is_degraded_and_never_active():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a", "host-b"))
coordinator = VpcCoordinator(store, fabric)
result = asyncio.run(coordinator.apply(payload(
hosts=[
{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a"},
]},
{"host_id": "host-b", "members": [
{"member_id": "member-b", "instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a"},
]},
],
)))
digest = result["desired_digest"]
observed = asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert observed["state"] == "degraded"
assert observed["next_stage_allowed"] is False
with pytest.raises(VpcError) as inconsistent:
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": False,
"private_dataplane_verified": True},
"host-b": {"state": "degraded", "observed_revision": 0,
"observed_digest": None, "transport_verified": False},
}))
assert inconsistent.value.error_code == "vpc_observation_invalid"
def test_coordinator_rejects_cross_region_fabric_even_when_revision_is_active():
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def wrong_region(fabric_id):
status = original(fabric_id)
status["region"] = "us"
return status
fabric.get_status = wrong_region
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == "vpc_fabric_region_mismatch"
@pytest.mark.parametrize("mutation", [
lambda status: status.update(fabric_id="other-fabric"),
lambda status: status.update(revision=True),
])
def test_fabric_gate_rejects_tampered_identity_or_revision(mutation):
store = MemoryStore()
fabric = FakeFabric()
original = fabric.get_status
def tampered(fabric_id):
status = original(fabric_id)
mutation(status)
return status
fabric.get_status = tampered
result = asyncio.run(VpcCoordinator(store, fabric).apply(payload()))
assert result["next_stage_allowed"] is False
assert result["state"] == "degraded"
def test_coordinator_revision_conflict_and_tampered_record_fail_closed():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
first = asyncio.run(coordinator.apply(payload()))
replay = asyncio.run(coordinator.apply(payload()))
assert replay["desired_digest"] == first["desired_digest"]
with pytest.raises(VpcError) as error:
asyncio.run(coordinator.apply(payload(
operation_id="vpc-op-00000002",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert error.value.error_code == "vpc_revision_conflict"
with pytest.raises(VpcError) as invalid_revision:
asyncio.run(coordinator.apply(payload(
desired_revision=0, operation_id="vpc-op-00000003",
)))
assert invalid_revision.value.error_code == "vpc_invalid"
asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)))
with pytest.raises(VpcError) as stale:
asyncio.run(coordinator.apply(payload(
desired_revision=1, operation_id="vpc-op-00000005",
members=[{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"host_id": "host-a", "private_ip": "10.20.0.12",
"subnet_id": "subnet-a"}],
hosts=[{"host_id": "host-a", "members": [
{"member_id": "member-a", "instance_uuid": INSTANCE_A,
"private_ip": "10.20.0.12", "subnet_id": "subnet-a"},
]}],
)))
assert stale.value.error_code == "vpc_revision_conflict"
tampered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
tampered["desired_digest"] = "0" * 64
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as corrupt:
asyncio.run(coordinator.reconcile("vpc-hk-1"))
assert corrupt.value.error_code == "vpc_record_corrupt"
tampered["vpc_id"] = "other-vpc"
tampered["desired_digest"] = first["desired_digest"]
store.set("vpc_desired", "vpc-hk-1", json.dumps(tampered))
with pytest.raises(VpcError) as mismatched:
coordinator.get_status("vpc-hk-1")
assert mismatched.value.error_code == "vpc_record_corrupt"
# Restore a structurally valid root and corrupt only observed evidence;
# a newer target must not overwrite that evidence silently.
restored = json.loads(store.get("vpc_desired", "vpc-hk-1"))
restored["vpc_id"] = "vpc-hk-1"
restored["desired_revision"] = 2
restored["desired_digest"] = parse_vpc_plan(payload(
desired_revision=2, operation_id="vpc-op-00000004",
)).desired_digest
restored["observed_revision"] = 99
store.set("vpc_desired", "vpc-hk-1", json.dumps(restored))
with pytest.raises(VpcError) as observed_corrupt:
asyncio.run(coordinator.apply(payload(
desired_revision=3, operation_id="vpc-op-00000006",
)))
assert observed_corrupt.value.error_code == "vpc_record_corrupt"
def test_recover_invalidates_observed_gate_and_record_round_trips():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
raw = json.loads(store.get("vpc_desired", "vpc-hk-1"))
raw["state"] = "active"
raw["next_stage_allowed"] = True
raw["observed_revision"] = 1
raw["observed_digest"] = raw["desired_digest"]
raw["hosts"][0].update(state="active", observed_revision=1,
observed_digest=raw["desired_digest"],
transport_verified=True,
private_dataplane_verified=True)
store.set("vpc_desired", "vpc-hk-1", json.dumps(raw))
coordinator.recover()
recovered = json.loads(store.get("vpc_desired", "vpc-hk-1"))
assert recovered["state"] == "uncertain"
assert recovered["next_stage_allowed"] is False
assert recovered["observed_revision"] == 1
assert recovered["observed_digest"] == recovered["desired_digest"]
assert recovered["fabric_gate"]["active"] is False
assert recovered["fabric_gate"]["error_code"] == "vpc_recovery_pending"
assert recovered["hosts"][0]["observed_revision"] == 1
assert _plan_from_record(recovered).vpc_id == "vpc-hk-1"
def test_removed_children_are_tombstoned_until_observed_cleanup():
store = MemoryStore()
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
asyncio.run(coordinator.apply(payload()))
changed = payload(
desired_revision=2,
operation_id="vpc-op-00000002",
hosts=[{"host_id": "host-a", "members": []}],
members=[],
)
asyncio.run(coordinator.apply(changed))
old = json.loads(store.get("vpc_member", "vpc-hk-1.member-a"))
assert old["state"] == "removed"
assert old["removed"] is True
def test_new_desired_revision_keeps_last_observed_evidence_until_readback():
store = MemoryStore()
fabric = FakeFabric(hosts=("host-a",))
coordinator = VpcCoordinator(store, fabric)
first = asyncio.run(coordinator.apply(payload()))
digest = first["desired_digest"]
asyncio.run(coordinator.observe("vpc-hk-1", {
"host-a": {"state": "active", "observed_revision": 1,
"observed_digest": digest, "transport_verified": True,
"private_dataplane_verified": True},
}))
second = asyncio.run(coordinator.apply(payload(
desired_revision=2, operation_id="vpc-op-00000002",
)))
assert second["state"] == "applying"
assert second["next_stage_allowed"] is False
assert second["observed_revision"] == 1
assert second["observed_digest"] == digest
assert second["hosts"][0]["observed_revision"] == 1