You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 lines
12 KiB
Python
352 lines
12 KiB
Python
"""Confirmed VPC writes must not poison later deterministic retry paths."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from virsh_panel import vpc_dataplane as dataplane
|
|
from virsh_panel.coordination import mark_remote_write_started
|
|
from virsh_panel.fabric import FabricNode, FabricPlan
|
|
from virsh_panel.vpc import VpcCoordinator, parse_vpc_plan
|
|
from virsh_panel.vpc_dataplane import (
|
|
DATAPLANE_OWNER_MARKER,
|
|
VpcPrivateHostAdapter,
|
|
derive_private_mac,
|
|
)
|
|
from test_engine_coordination import request, running_engine
|
|
from test_vpc import FakeFabric, INSTANCE_A, INSTANCE_B, payload
|
|
|
|
|
|
FABRIC_DIGEST = "f" * 64
|
|
NFT_RECEIPT = ",".join(character * 64 for character in "abc")
|
|
WG_KEYS = (
|
|
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=",
|
|
"AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=",
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def empty_apply_receipts():
|
|
with dataplane._APPLY_RECEIPT_LOCK:
|
|
dataplane._APPLY_RECEIPTS.clear()
|
|
yield
|
|
with dataplane._APPLY_RECEIPT_LOCK:
|
|
dataplane._APPLY_RECEIPTS.clear()
|
|
|
|
|
|
def _target(*, two_hosts: bool = False):
|
|
hosts = [{
|
|
"host_id": "host-a",
|
|
"members": [{
|
|
"member_id": "member-a",
|
|
"instance_uuid": INSTANCE_A,
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
}]
|
|
if two_hosts:
|
|
hosts.append({
|
|
"host_id": "host-b",
|
|
"members": [{
|
|
"member_id": "member-b",
|
|
"instance_uuid": INSTANCE_B,
|
|
"private_ip_cidr": "10.20.0.11/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
})
|
|
return payload(hosts=hosts)
|
|
|
|
|
|
def _profile(host_id: str) -> dict[str, object]:
|
|
return {
|
|
"host_id": host_id,
|
|
"host": f"{host_id}.example.test",
|
|
"port": 22,
|
|
"fingerprint": f"host-key-{host_id}",
|
|
"ssh_target": host_id,
|
|
"ssh_config_file": f"/test/{host_id}/ssh_config",
|
|
"storage_pool": "images",
|
|
"iso_pool": "iso",
|
|
"rescue_iso_pool": "rescue",
|
|
"default_network": "default",
|
|
"libvirt_uri": "fake",
|
|
}
|
|
|
|
|
|
def _fabric_plan(host_ids: tuple[str, ...]) -> FabricPlan:
|
|
return FabricPlan(
|
|
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
|
tuple(
|
|
FabricNode(
|
|
host_id,
|
|
f"100.64.0.{index + 1}",
|
|
f"192.0.2.{index + 1}:51820",
|
|
51820,
|
|
)
|
|
for index, host_id in enumerate(host_ids)
|
|
),
|
|
FABRIC_DIGEST,
|
|
)
|
|
|
|
|
|
def _confirmed_host(plan, host_id: str) -> dict[str, object]:
|
|
member = next(member for member in plan.members
|
|
if member.host_id == host_id)
|
|
return {
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"private_dataplane_verified": True,
|
|
"config_verified": True,
|
|
"host_config_reusable": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"member_nics": [{
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": derive_private_mac(
|
|
plan.vpc_id, member.member_id),
|
|
"config_verified": True,
|
|
"xml_verified": True,
|
|
"seed_verified": True,
|
|
"domain_running": False,
|
|
"guest_network_verified": False,
|
|
"guest_network_readback": "stopped",
|
|
"guest_reason_code": "vpc_private_guest_stopped",
|
|
"restart_required": False,
|
|
}],
|
|
"routes": [],
|
|
"wg_allowed_ips": [],
|
|
"reason_code": None,
|
|
}
|
|
|
|
|
|
def _durable_vpc(store) -> dict[str, object]:
|
|
return json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
|
|
|
|
def _install_coordinator(engine, store, target, monkeypatch):
|
|
plan = parse_vpc_plan(target)
|
|
host_ids = tuple(host.host_id for host in plan.hosts)
|
|
fabric_plan = _fabric_plan(host_ids)
|
|
|
|
class TestFabric(FakeFabric):
|
|
@staticmethod
|
|
def _plan_from_record(_record):
|
|
return fabric_plan
|
|
|
|
async def _components(self, selected):
|
|
return {
|
|
node.host_id: (
|
|
SimpleNamespace(manager=object()),
|
|
_profile(node.host_id),
|
|
)
|
|
for node in selected.nodes
|
|
}
|
|
|
|
coordinator = VpcCoordinator(
|
|
store,
|
|
TestFabric(hosts=host_ids),
|
|
registry=object(),
|
|
catalog_lock=engine._network_catalog_mutation_lock,
|
|
vm_lock=engine._target_lock,
|
|
coordinator=engine.coordinator,
|
|
)
|
|
coordinator._validate_dataplane_resources = lambda _plan: None
|
|
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
|
|
public_keys = {
|
|
host_id: WG_KEYS[index]
|
|
for index, host_id in enumerate(host_ids)
|
|
}
|
|
coordinator._fabric_record = lambda _plan: {
|
|
"public_keys": public_keys,
|
|
}
|
|
coordinator._fabric_peers_for_plan = lambda _plan, _record: [
|
|
{
|
|
"host_id": host_id,
|
|
"public_key": public_keys[host_id],
|
|
"overlay_ip": f"100.64.0.{index + 1}",
|
|
"tenant_allowed_ips": sorted(
|
|
f"{member.private_ip}/32"
|
|
for member in plan.members
|
|
if member.host_id == host_id
|
|
),
|
|
}
|
|
for index, host_id in enumerate(host_ids)
|
|
]
|
|
coordinator._require_host_member_mutations_stopped = (
|
|
lambda *_args, **_kwargs: None)
|
|
engine.vpc = coordinator
|
|
|
|
writes: list[str] = []
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter,
|
|
"_deployment_contract",
|
|
lambda self: (
|
|
"networkmanager_nmcli", "/usr/sbin/ifup",
|
|
"/usr/sbin/ifdown", "/usr/sbin/ifquery",
|
|
"native_nftables",
|
|
),
|
|
)
|
|
|
|
def strict_receipt(self, script):
|
|
assert "__VC_VPC_APPLIED__=yes" in script
|
|
host_id = str(self.profile["host_id"])
|
|
writes.append(host_id)
|
|
return "\n".join((
|
|
"__VC_VPC_APPLIED__=yes",
|
|
f"__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}",
|
|
f"__VC_VPC_REVISION__={plan.desired_revision}",
|
|
f"__VC_VPC_DIGEST__={plan.desired_digest}",
|
|
f"__VC_VPC_NFT_RECEIPT__={NFT_RECEIPT}",
|
|
))
|
|
|
|
def confirmed_observation(self, host_plan, *, xml_by_instance=None):
|
|
assert xml_by_instance is None or set(xml_by_instance) == {
|
|
member.instance_uuid
|
|
for member in plan.members
|
|
if member.host_id == host_plan.host_id
|
|
}
|
|
return _confirmed_host(plan, host_plan.host_id), True, NFT_RECEIPT
|
|
|
|
monkeypatch.setattr(VpcPrivateHostAdapter, "_run", strict_receipt)
|
|
monkeypatch.setattr(
|
|
VpcPrivateHostAdapter, "_observe", confirmed_observation)
|
|
return coordinator, plan, writes
|
|
|
|
|
|
def test_confirmed_ssh_apply_then_deterministic_libvirt_failure_releases_family(
|
|
tmp_path, monkeypatch):
|
|
from virsh_panel.libvirt_service import VMError, VMService
|
|
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
target = _target()
|
|
coordinator, plan, writes = _install_coordinator(
|
|
engine, store, target, monkeypatch)
|
|
attempts: list[str] = []
|
|
failing = {"host-a"}
|
|
|
|
monkeypatch.setattr(
|
|
VMService, "require_vpc_members_stopped",
|
|
lambda *_args, **_kwargs: None,
|
|
)
|
|
|
|
def configure(self, instance_uuid, **_kwargs):
|
|
attempts.append(self.execution_scope)
|
|
with self._operation_guard(
|
|
instance_uuid, storage=True,
|
|
tracked_mutations=True):
|
|
if self.execution_scope in failing:
|
|
raise VMError(
|
|
"native snapshot state read failed before libvirt write",
|
|
502,
|
|
error_code="vpc_vm_native_snapshot_state_unknown",
|
|
)
|
|
mark_remote_write_started()
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}",
|
|
"live": None,
|
|
"seed_verified": True,
|
|
"restart_required": False,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
VMService, "ensure_private_interface",
|
|
configure,
|
|
)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target,
|
|
operation_id=target["operation_id"])
|
|
|
|
assert status == 200, result
|
|
assert attempts == ["host-a"], json.dumps(result, sort_keys=True)
|
|
assert writes == ["host-a"]
|
|
durable = _durable_vpc(store)
|
|
assert durable["hosts"][0]["last_error_code"] \
|
|
== "vpc_vm_native_snapshot_state_unknown"
|
|
assert durable["hosts"][0]["state"] == "uncertain"
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
failing.clear()
|
|
attempts.clear()
|
|
retry_status, retry_result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target,
|
|
operation_id=target["operation_id"])
|
|
assert retry_status == 200, retry_result
|
|
assert attempts == ["host-a"]
|
|
assert writes == ["host-a"]
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_confirmed_host_a_then_host_b_prewrite_failure_releases_family(
|
|
tmp_path, monkeypatch):
|
|
from virsh_panel.libvirt_service import VMError, VMService
|
|
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
target = _target(two_hosts=True)
|
|
coordinator, plan, writes = _install_coordinator(
|
|
engine, store, target, monkeypatch)
|
|
attempts: list[str] = []
|
|
failing = {"host-b"}
|
|
|
|
monkeypatch.setattr(
|
|
VMService, "require_vpc_members_stopped",
|
|
lambda *_args, **_kwargs: None,
|
|
)
|
|
|
|
def configure(self, instance_uuid, **_kwargs):
|
|
attempts.append(self.execution_scope)
|
|
with self._operation_guard(
|
|
instance_uuid, storage=True,
|
|
tracked_mutations=True):
|
|
if self.execution_scope in failing:
|
|
raise VMError(
|
|
"host B snapshot state read failed before libvirt write",
|
|
502,
|
|
error_code="vpc_vm_native_snapshot_state_unknown",
|
|
)
|
|
mark_remote_write_started()
|
|
return {
|
|
"xml": f"inactive:{instance_uuid}",
|
|
"live": None,
|
|
"seed_verified": True,
|
|
"restart_required": False,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
VMService, "ensure_private_interface", configure)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target,
|
|
operation_id=target["operation_id"])
|
|
|
|
assert status == 200, result
|
|
assert attempts == ["host-a", "host-b"], json.dumps(
|
|
result, sort_keys=True)
|
|
assert writes == ["host-a", "host-b"]
|
|
durable = _durable_vpc(store)
|
|
by_host = {host["host_id"]: host for host in durable["hosts"]}
|
|
assert coordinator._confirmed_active_host(
|
|
by_host["host-a"], plan, allow_pending=True)
|
|
assert by_host["host-b"]["last_error_code"] \
|
|
== "vpc_vm_native_snapshot_state_unknown"
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
failing.clear()
|
|
attempts.clear()
|
|
retry_status, retry_result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target,
|
|
operation_id=target["operation_id"])
|
|
assert retry_status == 200, retry_result
|
|
assert attempts == ["host-b"]
|
|
assert writes == ["host-a", "host-b"]
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|