You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_confirmed_write_re...

352 lines
12 KiB
Python

"""Confirmed VPC writes must not poison later deterministic retry paths."""
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
import pytest
from virsh_panel import vpc_dataplane as dataplane
from virsh_panel.coordination import mark_remote_write_started
from virsh_panel.fabric import FabricNode, FabricPlan
from virsh_panel.vpc import VpcCoordinator, parse_vpc_plan
from virsh_panel.vpc_dataplane import (
DATAPLANE_OWNER_MARKER,
VpcPrivateHostAdapter,
derive_private_mac,
)
from test_engine_coordination import request, running_engine
from test_vpc import FakeFabric, INSTANCE_A, INSTANCE_B, payload
FABRIC_DIGEST = "f" * 64
NFT_RECEIPT = ",".join(character * 64 for character in "abc")
WG_KEYS = (
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=",
"AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=",
)
@pytest.fixture(autouse=True)
def empty_apply_receipts():
with dataplane._APPLY_RECEIPT_LOCK:
dataplane._APPLY_RECEIPTS.clear()
yield
with dataplane._APPLY_RECEIPT_LOCK:
dataplane._APPLY_RECEIPTS.clear()
def _target(*, two_hosts: bool = False):
hosts = [{
"host_id": "host-a",
"members": [{
"member_id": "member-a",
"instance_uuid": INSTANCE_A,
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
}]
if two_hosts:
hosts.append({
"host_id": "host-b",
"members": [{
"member_id": "member-b",
"instance_uuid": INSTANCE_B,
"private_ip_cidr": "10.20.0.11/32",
"subnet_id": "subnet-a",
}],
})
return payload(hosts=hosts)
def _profile(host_id: str) -> dict[str, object]:
return {
"host_id": host_id,
"host": f"{host_id}.example.test",
"port": 22,
"fingerprint": f"host-key-{host_id}",
"ssh_target": host_id,
"ssh_config_file": f"/test/{host_id}/ssh_config",
"storage_pool": "images",
"iso_pool": "iso",
"rescue_iso_pool": "rescue",
"default_network": "default",
"libvirt_uri": "fake",
}
def _fabric_plan(host_ids: tuple[str, ...]) -> FabricPlan:
return FabricPlan(
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
tuple(
FabricNode(
host_id,
f"100.64.0.{index + 1}",
f"192.0.2.{index + 1}:51820",
51820,
)
for index, host_id in enumerate(host_ids)
),
FABRIC_DIGEST,
)
def _confirmed_host(plan, host_id: str) -> dict[str, object]:
member = next(member for member in plan.members
if member.host_id == host_id)
return {
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"config_verified": True,
"host_config_reusable": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"member_nics": [{
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": derive_private_mac(
plan.vpc_id, member.member_id),
"config_verified": True,
"xml_verified": True,
"seed_verified": True,
"domain_running": False,
"guest_network_verified": False,
"guest_network_readback": "stopped",
"guest_reason_code": "vpc_private_guest_stopped",
"restart_required": False,
}],
"routes": [],
"wg_allowed_ips": [],
"reason_code": None,
}
def _durable_vpc(store) -> dict[str, object]:
return json.loads(store.get("vpc_desired", "vpc-hk-1"))
def _install_coordinator(engine, store, target, monkeypatch):
plan = parse_vpc_plan(target)
host_ids = tuple(host.host_id for host in plan.hosts)
fabric_plan = _fabric_plan(host_ids)
class TestFabric(FakeFabric):
@staticmethod
def _plan_from_record(_record):
return fabric_plan
async def _components(self, selected):
return {
node.host_id: (
SimpleNamespace(manager=object()),
_profile(node.host_id),
)
for node in selected.nodes
}
coordinator = VpcCoordinator(
store,
TestFabric(hosts=host_ids),
registry=object(),
catalog_lock=engine._network_catalog_mutation_lock,
vm_lock=engine._target_lock,
coordinator=engine.coordinator,
)
coordinator._validate_dataplane_resources = lambda _plan: None
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
public_keys = {
host_id: WG_KEYS[index]
for index, host_id in enumerate(host_ids)
}
coordinator._fabric_record = lambda _plan: {
"public_keys": public_keys,
}
coordinator._fabric_peers_for_plan = lambda _plan, _record: [
{
"host_id": host_id,
"public_key": public_keys[host_id],
"overlay_ip": f"100.64.0.{index + 1}",
"tenant_allowed_ips": sorted(
f"{member.private_ip}/32"
for member in plan.members
if member.host_id == host_id
),
}
for index, host_id in enumerate(host_ids)
]
coordinator._require_host_member_mutations_stopped = (
lambda *_args, **_kwargs: None)
engine.vpc = coordinator
writes: list[str] = []
monkeypatch.setattr(
VpcPrivateHostAdapter,
"_deployment_contract",
lambda self: (
"networkmanager_nmcli", "/usr/sbin/ifup",
"/usr/sbin/ifdown", "/usr/sbin/ifquery",
"native_nftables",
),
)
def strict_receipt(self, script):
assert "__VC_VPC_APPLIED__=yes" in script
host_id = str(self.profile["host_id"])
writes.append(host_id)
return "\n".join((
"__VC_VPC_APPLIED__=yes",
f"__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}",
f"__VC_VPC_REVISION__={plan.desired_revision}",
f"__VC_VPC_DIGEST__={plan.desired_digest}",
f"__VC_VPC_NFT_RECEIPT__={NFT_RECEIPT}",
))
def confirmed_observation(self, host_plan, *, xml_by_instance=None):
assert xml_by_instance is None or set(xml_by_instance) == {
member.instance_uuid
for member in plan.members
if member.host_id == host_plan.host_id
}
return _confirmed_host(plan, host_plan.host_id), True, NFT_RECEIPT
monkeypatch.setattr(VpcPrivateHostAdapter, "_run", strict_receipt)
monkeypatch.setattr(
VpcPrivateHostAdapter, "_observe", confirmed_observation)
return coordinator, plan, writes
def test_confirmed_ssh_apply_then_deterministic_libvirt_failure_releases_family(
tmp_path, monkeypatch):
from virsh_panel.libvirt_service import VMError, VMService
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
target = _target()
coordinator, plan, writes = _install_coordinator(
engine, store, target, monkeypatch)
attempts: list[str] = []
failing = {"host-a"}
monkeypatch.setattr(
VMService, "require_vpc_members_stopped",
lambda *_args, **_kwargs: None,
)
def configure(self, instance_uuid, **_kwargs):
attempts.append(self.execution_scope)
with self._operation_guard(
instance_uuid, storage=True,
tracked_mutations=True):
if self.execution_scope in failing:
raise VMError(
"native snapshot state read failed before libvirt write",
502,
error_code="vpc_vm_native_snapshot_state_unknown",
)
mark_remote_write_started()
return {
"xml": f"inactive:{instance_uuid}",
"live": None,
"seed_verified": True,
"restart_required": False,
}
monkeypatch.setattr(
VMService, "ensure_private_interface",
configure,
)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/apply", target,
operation_id=target["operation_id"])
assert status == 200, result
assert attempts == ["host-a"], json.dumps(result, sort_keys=True)
assert writes == ["host-a"]
durable = _durable_vpc(store)
assert durable["hosts"][0]["last_error_code"] \
== "vpc_vm_native_snapshot_state_unknown"
assert durable["hosts"][0]["state"] == "uncertain"
assert engine.coordinator.snapshot() == {}
failing.clear()
attempts.clear()
retry_status, retry_result = await request(
engine, "POST", "/internal/v1/vpcs/apply", target,
operation_id=target["operation_id"])
assert retry_status == 200, retry_result
assert attempts == ["host-a"]
assert writes == ["host-a"]
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
def test_confirmed_host_a_then_host_b_prewrite_failure_releases_family(
tmp_path, monkeypatch):
from virsh_panel.libvirt_service import VMError, VMService
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
target = _target(two_hosts=True)
coordinator, plan, writes = _install_coordinator(
engine, store, target, monkeypatch)
attempts: list[str] = []
failing = {"host-b"}
monkeypatch.setattr(
VMService, "require_vpc_members_stopped",
lambda *_args, **_kwargs: None,
)
def configure(self, instance_uuid, **_kwargs):
attempts.append(self.execution_scope)
with self._operation_guard(
instance_uuid, storage=True,
tracked_mutations=True):
if self.execution_scope in failing:
raise VMError(
"host B snapshot state read failed before libvirt write",
502,
error_code="vpc_vm_native_snapshot_state_unknown",
)
mark_remote_write_started()
return {
"xml": f"inactive:{instance_uuid}",
"live": None,
"seed_verified": True,
"restart_required": False,
}
monkeypatch.setattr(
VMService, "ensure_private_interface", configure)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/apply", target,
operation_id=target["operation_id"])
assert status == 200, result
assert attempts == ["host-a", "host-b"], json.dumps(
result, sort_keys=True)
assert writes == ["host-a", "host-b"]
durable = _durable_vpc(store)
by_host = {host["host_id"]: host for host in durable["hosts"]}
assert coordinator._confirmed_active_host(
by_host["host-a"], plan, allow_pending=True)
assert by_host["host-b"]["last_error_code"] \
== "vpc_vm_native_snapshot_state_unknown"
assert engine.coordinator.snapshot() == {}
failing.clear()
attempts.clear()
retry_status, retry_result = await request(
engine, "POST", "/internal/v1/vpcs/apply", target,
operation_id=target["operation_id"])
assert retry_status == 200, retry_result
assert attempts == ["host-b"]
assert writes == ["host-a", "host-b"]
assert engine.coordinator.snapshot() == {}
asyncio.run(run())