refactor(vpc): make membership reconciliation durable
parent
62653d7003
commit
d429699172
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,351 @@
|
||||
"""Confirmed VPC writes must not poison later deterministic retry paths."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel import vpc_dataplane as dataplane
|
||||
from virsh_panel.coordination import mark_remote_write_started
|
||||
from virsh_panel.fabric import FabricNode, FabricPlan
|
||||
from virsh_panel.vpc import VpcCoordinator, parse_vpc_plan
|
||||
from virsh_panel.vpc_dataplane import (
|
||||
DATAPLANE_OWNER_MARKER,
|
||||
VpcPrivateHostAdapter,
|
||||
derive_private_mac,
|
||||
)
|
||||
from test_engine_coordination import request, running_engine
|
||||
from test_vpc import FakeFabric, INSTANCE_A, INSTANCE_B, payload
|
||||
|
||||
|
||||
FABRIC_DIGEST = "f" * 64
|
||||
NFT_RECEIPT = ",".join(character * 64 for character in "abc")
|
||||
WG_KEYS = (
|
||||
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=",
|
||||
"AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def empty_apply_receipts():
|
||||
with dataplane._APPLY_RECEIPT_LOCK:
|
||||
dataplane._APPLY_RECEIPTS.clear()
|
||||
yield
|
||||
with dataplane._APPLY_RECEIPT_LOCK:
|
||||
dataplane._APPLY_RECEIPTS.clear()
|
||||
|
||||
|
||||
def _target(*, two_hosts: bool = False):
|
||||
hosts = [{
|
||||
"host_id": "host-a",
|
||||
"members": [{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": INSTANCE_A,
|
||||
"private_ip_cidr": "10.20.0.10/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}],
|
||||
}]
|
||||
if two_hosts:
|
||||
hosts.append({
|
||||
"host_id": "host-b",
|
||||
"members": [{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"private_ip_cidr": "10.20.0.11/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}],
|
||||
})
|
||||
return payload(hosts=hosts)
|
||||
|
||||
|
||||
def _profile(host_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"host_id": host_id,
|
||||
"host": f"{host_id}.example.test",
|
||||
"port": 22,
|
||||
"fingerprint": f"host-key-{host_id}",
|
||||
"ssh_target": host_id,
|
||||
"ssh_config_file": f"/test/{host_id}/ssh_config",
|
||||
"storage_pool": "images",
|
||||
"iso_pool": "iso",
|
||||
"rescue_iso_pool": "rescue",
|
||||
"default_network": "default",
|
||||
"libvirt_uri": "fake",
|
||||
}
|
||||
|
||||
|
||||
def _fabric_plan(host_ids: tuple[str, ...]) -> FabricPlan:
|
||||
return FabricPlan(
|
||||
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
||||
tuple(
|
||||
FabricNode(
|
||||
host_id,
|
||||
f"100.64.0.{index + 1}",
|
||||
f"192.0.2.{index + 1}:51820",
|
||||
51820,
|
||||
)
|
||||
for index, host_id in enumerate(host_ids)
|
||||
),
|
||||
FABRIC_DIGEST,
|
||||
)
|
||||
|
||||
|
||||
def _confirmed_host(plan, host_id: str) -> dict[str, object]:
|
||||
member = next(member for member in plan.members
|
||||
if member.host_id == host_id)
|
||||
return {
|
||||
"state": "active",
|
||||
"transport_verified": True,
|
||||
"private_dataplane_verified": True,
|
||||
"config_verified": True,
|
||||
"host_config_reusable": True,
|
||||
"config_observed_revision": plan.desired_revision,
|
||||
"config_observed_digest": plan.desired_digest,
|
||||
"member_nics": [{
|
||||
"member_id": member.member_id,
|
||||
"instance_uuid": member.instance_uuid,
|
||||
"private_ip_cidr": f"{member.private_ip}/32",
|
||||
"private_mac": derive_private_mac(
|
||||
plan.vpc_id, member.member_id),
|
||||
"config_verified": True,
|
||||
"xml_verified": True,
|
||||
"seed_verified": True,
|
||||
"domain_running": False,
|
||||
"guest_network_verified": False,
|
||||
"guest_network_readback": "stopped",
|
||||
"guest_reason_code": "vpc_private_guest_stopped",
|
||||
"restart_required": False,
|
||||
}],
|
||||
"routes": [],
|
||||
"wg_allowed_ips": [],
|
||||
"reason_code": None,
|
||||
}
|
||||
|
||||
|
||||
def _durable_vpc(store) -> dict[str, object]:
|
||||
return json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
||||
|
||||
|
||||
def _install_coordinator(engine, store, target, monkeypatch):
|
||||
plan = parse_vpc_plan(target)
|
||||
host_ids = tuple(host.host_id for host in plan.hosts)
|
||||
fabric_plan = _fabric_plan(host_ids)
|
||||
|
||||
class TestFabric(FakeFabric):
|
||||
@staticmethod
|
||||
def _plan_from_record(_record):
|
||||
return fabric_plan
|
||||
|
||||
async def _components(self, selected):
|
||||
return {
|
||||
node.host_id: (
|
||||
SimpleNamespace(manager=object()),
|
||||
_profile(node.host_id),
|
||||
)
|
||||
for node in selected.nodes
|
||||
}
|
||||
|
||||
coordinator = VpcCoordinator(
|
||||
store,
|
||||
TestFabric(hosts=host_ids),
|
||||
registry=object(),
|
||||
catalog_lock=engine._network_catalog_mutation_lock,
|
||||
vm_lock=engine._target_lock,
|
||||
coordinator=engine.coordinator,
|
||||
)
|
||||
coordinator._validate_dataplane_resources = lambda _plan: None
|
||||
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
|
||||
public_keys = {
|
||||
host_id: WG_KEYS[index]
|
||||
for index, host_id in enumerate(host_ids)
|
||||
}
|
||||
coordinator._fabric_record = lambda _plan: {
|
||||
"public_keys": public_keys,
|
||||
}
|
||||
coordinator._fabric_peers_for_plan = lambda _plan, _record: [
|
||||
{
|
||||
"host_id": host_id,
|
||||
"public_key": public_keys[host_id],
|
||||
"overlay_ip": f"100.64.0.{index + 1}",
|
||||
"tenant_allowed_ips": sorted(
|
||||
f"{member.private_ip}/32"
|
||||
for member in plan.members
|
||||
if member.host_id == host_id
|
||||
),
|
||||
}
|
||||
for index, host_id in enumerate(host_ids)
|
||||
]
|
||||
coordinator._require_host_member_mutations_stopped = (
|
||||
lambda *_args, **_kwargs: None)
|
||||
engine.vpc = coordinator
|
||||
|
||||
writes: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
VpcPrivateHostAdapter,
|
||||
"_deployment_contract",
|
||||
lambda self: (
|
||||
"networkmanager_nmcli", "/usr/sbin/ifup",
|
||||
"/usr/sbin/ifdown", "/usr/sbin/ifquery",
|
||||
"native_nftables",
|
||||
),
|
||||
)
|
||||
|
||||
def strict_receipt(self, script):
|
||||
assert "__VC_VPC_APPLIED__=yes" in script
|
||||
host_id = str(self.profile["host_id"])
|
||||
writes.append(host_id)
|
||||
return "\n".join((
|
||||
"__VC_VPC_APPLIED__=yes",
|
||||
f"__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}",
|
||||
f"__VC_VPC_REVISION__={plan.desired_revision}",
|
||||
f"__VC_VPC_DIGEST__={plan.desired_digest}",
|
||||
f"__VC_VPC_NFT_RECEIPT__={NFT_RECEIPT}",
|
||||
))
|
||||
|
||||
def confirmed_observation(self, host_plan, *, xml_by_instance=None):
|
||||
assert xml_by_instance is None or set(xml_by_instance) == {
|
||||
member.instance_uuid
|
||||
for member in plan.members
|
||||
if member.host_id == host_plan.host_id
|
||||
}
|
||||
return _confirmed_host(plan, host_plan.host_id), True, NFT_RECEIPT
|
||||
|
||||
monkeypatch.setattr(VpcPrivateHostAdapter, "_run", strict_receipt)
|
||||
monkeypatch.setattr(
|
||||
VpcPrivateHostAdapter, "_observe", confirmed_observation)
|
||||
return coordinator, plan, writes
|
||||
|
||||
|
||||
def test_confirmed_ssh_apply_then_deterministic_libvirt_failure_releases_family(
|
||||
tmp_path, monkeypatch):
|
||||
from virsh_panel.libvirt_service import VMError, VMService
|
||||
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
target = _target()
|
||||
coordinator, plan, writes = _install_coordinator(
|
||||
engine, store, target, monkeypatch)
|
||||
attempts: list[str] = []
|
||||
failing = {"host-a"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
VMService, "require_vpc_members_stopped",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
def configure(self, instance_uuid, **_kwargs):
|
||||
attempts.append(self.execution_scope)
|
||||
with self._operation_guard(
|
||||
instance_uuid, storage=True,
|
||||
tracked_mutations=True):
|
||||
if self.execution_scope in failing:
|
||||
raise VMError(
|
||||
"native snapshot state read failed before libvirt write",
|
||||
502,
|
||||
error_code="vpc_vm_native_snapshot_state_unknown",
|
||||
)
|
||||
mark_remote_write_started()
|
||||
return {
|
||||
"xml": f"inactive:{instance_uuid}",
|
||||
"live": None,
|
||||
"seed_verified": True,
|
||||
"restart_required": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
VMService, "ensure_private_interface",
|
||||
configure,
|
||||
)
|
||||
status, result = await request(
|
||||
engine, "POST", "/internal/v1/vpcs/apply", target,
|
||||
operation_id=target["operation_id"])
|
||||
|
||||
assert status == 200, result
|
||||
assert attempts == ["host-a"], json.dumps(result, sort_keys=True)
|
||||
assert writes == ["host-a"]
|
||||
durable = _durable_vpc(store)
|
||||
assert durable["hosts"][0]["last_error_code"] \
|
||||
== "vpc_vm_native_snapshot_state_unknown"
|
||||
assert durable["hosts"][0]["state"] == "uncertain"
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
failing.clear()
|
||||
attempts.clear()
|
||||
retry_status, retry_result = await request(
|
||||
engine, "POST", "/internal/v1/vpcs/apply", target,
|
||||
operation_id=target["operation_id"])
|
||||
assert retry_status == 200, retry_result
|
||||
assert attempts == ["host-a"]
|
||||
assert writes == ["host-a"]
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_confirmed_host_a_then_host_b_prewrite_failure_releases_family(
|
||||
tmp_path, monkeypatch):
|
||||
from virsh_panel.libvirt_service import VMError, VMService
|
||||
|
||||
async def run():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
target = _target(two_hosts=True)
|
||||
coordinator, plan, writes = _install_coordinator(
|
||||
engine, store, target, monkeypatch)
|
||||
attempts: list[str] = []
|
||||
failing = {"host-b"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
VMService, "require_vpc_members_stopped",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
def configure(self, instance_uuid, **_kwargs):
|
||||
attempts.append(self.execution_scope)
|
||||
with self._operation_guard(
|
||||
instance_uuid, storage=True,
|
||||
tracked_mutations=True):
|
||||
if self.execution_scope in failing:
|
||||
raise VMError(
|
||||
"host B snapshot state read failed before libvirt write",
|
||||
502,
|
||||
error_code="vpc_vm_native_snapshot_state_unknown",
|
||||
)
|
||||
mark_remote_write_started()
|
||||
return {
|
||||
"xml": f"inactive:{instance_uuid}",
|
||||
"live": None,
|
||||
"seed_verified": True,
|
||||
"restart_required": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
VMService, "ensure_private_interface", configure)
|
||||
status, result = await request(
|
||||
engine, "POST", "/internal/v1/vpcs/apply", target,
|
||||
operation_id=target["operation_id"])
|
||||
|
||||
assert status == 200, result
|
||||
assert attempts == ["host-a", "host-b"], json.dumps(
|
||||
result, sort_keys=True)
|
||||
assert writes == ["host-a", "host-b"]
|
||||
durable = _durable_vpc(store)
|
||||
by_host = {host["host_id"]: host for host in durable["hosts"]}
|
||||
assert coordinator._confirmed_active_host(
|
||||
by_host["host-a"], plan, allow_pending=True)
|
||||
assert by_host["host-b"]["last_error_code"] \
|
||||
== "vpc_vm_native_snapshot_state_unknown"
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
failing.clear()
|
||||
attempts.clear()
|
||||
retry_status, retry_result = await request(
|
||||
engine, "POST", "/internal/v1/vpcs/apply", target,
|
||||
operation_id=target["operation_id"])
|
||||
assert retry_status == 200, retry_result
|
||||
assert attempts == ["host-b"]
|
||||
assert writes == ["host-a", "host-b"]
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
asyncio.run(run())
|
||||
@ -0,0 +1,674 @@
|
||||
"""Regression contracts for durable VPC revision admission and migration."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.fabric import FabricNode, FabricPlan
|
||||
from virsh_panel.vpc import (
|
||||
VPC_DELETE_RECEIPT_KIND,
|
||||
VpcCoordinator,
|
||||
VpcError,
|
||||
parse_vpc_delete_plan,
|
||||
parse_vpc_plan,
|
||||
)
|
||||
from virsh_panel.vpc_dataplane import derive_private_mac
|
||||
from test_vpc import (
|
||||
FakeFabric,
|
||||
INSTANCE_A,
|
||||
INSTANCE_B,
|
||||
MemoryStore,
|
||||
confirm_delete_hosts,
|
||||
delete_payload,
|
||||
payload,
|
||||
)
|
||||
|
||||
|
||||
GATE = {"active": True, "available": True}
|
||||
|
||||
|
||||
class SimulatedProcessInterruption(BaseException):
|
||||
"""Model a process exit after a host write has crossed its boundary."""
|
||||
|
||||
|
||||
def _two_host_payload(*, desired_revision=1, operation_id="vpc-op-00000001"):
|
||||
return payload(
|
||||
desired_revision=desired_revision,
|
||||
operation_id=operation_id,
|
||||
hosts=[
|
||||
{"host_id": "host-a", "members": [{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": INSTANCE_A,
|
||||
"private_ip_cidr": "10.20.0.10/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}]},
|
||||
{"host_id": "host-b", "members": [{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"private_ip_cidr": "10.20.0.11/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}]},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _exact_host(plan, host_id, *, pending=None):
|
||||
member = next(item for item in plan.members if item.host_id == host_id)
|
||||
return {
|
||||
"host_id": host_id,
|
||||
"vpc_id": plan.vpc_id,
|
||||
"member_ids": [member.member_id],
|
||||
"desired_revision": plan.desired_revision,
|
||||
"desired_digest": plan.desired_digest,
|
||||
"observed_revision": plan.desired_revision,
|
||||
"observed_digest": plan.desired_digest,
|
||||
"state": "active",
|
||||
"transport_verified": True,
|
||||
"private_dataplane_verified": True,
|
||||
"config_verified": True,
|
||||
"host_config_reusable": True,
|
||||
"config_observed_revision": plan.desired_revision,
|
||||
"config_observed_digest": plan.desired_digest,
|
||||
"retiring_members": [],
|
||||
**({"pending_member_mutations": list(pending)}
|
||||
if pending is not None else {}),
|
||||
"member_nics": [{
|
||||
"member_id": member.member_id,
|
||||
"instance_uuid": member.instance_uuid,
|
||||
"private_ip_cidr": f"{member.private_ip}/32",
|
||||
"private_mac": derive_private_mac(plan.vpc_id, member.member_id),
|
||||
"config_verified": True,
|
||||
"xml_verified": True,
|
||||
"seed_verified": True,
|
||||
"domain_running": False,
|
||||
"guest_network_verified": False,
|
||||
"guest_network_readback": "stopped",
|
||||
"guest_reason_code": "vpc_private_guest_stopped",
|
||||
"restart_required": False,
|
||||
}],
|
||||
"routes": [],
|
||||
"wg_allowed_ips": [],
|
||||
"readback": {},
|
||||
}
|
||||
|
||||
|
||||
def _decode(store, kind, key):
|
||||
raw = store.get(kind, key)
|
||||
return json.loads(raw) if raw is not None else None
|
||||
|
||||
|
||||
def test_new_revision_is_durable_with_children_before_first_host_write():
|
||||
store = MemoryStore()
|
||||
coordinator = VpcCoordinator(
|
||||
store, FakeFabric(hosts=("host-a", "host-b")))
|
||||
previous_plan = parse_vpc_plan(payload())
|
||||
previous = coordinator._desired_record(
|
||||
previous_plan, GATE, state="applying")
|
||||
coordinator._save("vpc_desired", previous_plan.vpc_id, previous)
|
||||
coordinator._save_children(previous)
|
||||
|
||||
plan = parse_vpc_plan(_two_host_payload(
|
||||
desired_revision=2, operation_id="vpc-op-00000002"))
|
||||
durable_at_write = {}
|
||||
host_writes = []
|
||||
|
||||
async def first_host_write_then_interrupt(*_args, **_kwargs):
|
||||
host_writes.append("started")
|
||||
durable_at_write.update({
|
||||
"root": _decode(store, "vpc_desired", plan.vpc_id),
|
||||
"member": _decode(
|
||||
store, "vpc_member", f"{plan.vpc_id}.member-b"),
|
||||
"host": _decode(store, "vpc_host", f"{plan.vpc_id}.host-b"),
|
||||
})
|
||||
raise SimulatedProcessInterruption
|
||||
|
||||
coordinator._reconcile_dataplane = first_host_write_then_interrupt
|
||||
|
||||
with pytest.raises(SimulatedProcessInterruption):
|
||||
asyncio.run(coordinator._apply_locked(plan))
|
||||
|
||||
assert host_writes == ["started"]
|
||||
assert durable_at_write["root"]["desired_revision"] == 2
|
||||
assert durable_at_write["root"]["desired_digest"] == plan.desired_digest
|
||||
assert durable_at_write["member"]["desired_revision"] == 2
|
||||
assert durable_at_write["member"]["instance_uuid"] == INSTANCE_B
|
||||
assert durable_at_write["host"]["desired_revision"] == 2
|
||||
assert durable_at_write["host"]["pending_member_mutations"] == [INSTANCE_B]
|
||||
|
||||
# A process exit cannot run an in-memory finalizer. The pre-write intent
|
||||
# must therefore remain sufficient for a fresh process to resume revision 2.
|
||||
assert _decode(store, "vpc_desired", plan.vpc_id)["desired_revision"] == 2
|
||||
assert _decode(
|
||||
store, "vpc_member", f"{plan.vpc_id}.member-b")[
|
||||
"desired_digest"] == plan.desired_digest
|
||||
|
||||
|
||||
def test_retry_preflights_and_locks_full_durable_mutation_closure():
|
||||
plan = parse_vpc_plan(_two_host_payload())
|
||||
fabric_plan = FabricPlan(
|
||||
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
||||
(
|
||||
FabricNode(
|
||||
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
||||
FabricNode(
|
||||
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
||||
),
|
||||
"f" * 64,
|
||||
)
|
||||
|
||||
class RetryFabric(FakeFabric):
|
||||
@staticmethod
|
||||
def _plan_from_record(_record):
|
||||
return fabric_plan
|
||||
|
||||
async def _components(self, _plan):
|
||||
return {
|
||||
host_id: (
|
||||
SimpleNamespace(manager=object()),
|
||||
{"host_id": host_id},
|
||||
)
|
||||
for host_id in ("host-a", "host-b")
|
||||
}
|
||||
|
||||
acquired = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def vm_lock(host_id, instance_uuid):
|
||||
acquired.append((host_id, instance_uuid))
|
||||
yield
|
||||
|
||||
coordinator = VpcCoordinator(
|
||||
MemoryStore(), RetryFabric(hosts=("host-a", "host-b")),
|
||||
registry=object(), vm_lock=vm_lock,
|
||||
)
|
||||
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
|
||||
coordinator._validate_dataplane_resources = lambda _plan: None
|
||||
coordinator._fabric_record = lambda _plan: {
|
||||
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
||||
}
|
||||
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
||||
|
||||
record = coordinator._desired_record(plan, GATE, state="applying")
|
||||
by_host = {host["host_id"]: host for host in record["hosts"]}
|
||||
# Host A has exact persistent configuration, so only host B belongs to the
|
||||
# next write selection. Its durable marker remains part of this revision's
|
||||
# stop-before-write closure until every selected host has converged.
|
||||
by_host["host-a"].update(
|
||||
_exact_host(plan, "host-a", pending=[INSTANCE_A]))
|
||||
checked = []
|
||||
host_writes = []
|
||||
|
||||
def require_stopped(_profile, host_id, instance_uuids):
|
||||
checked.append((host_id, tuple(instance_uuids)))
|
||||
if host_id == "host-a":
|
||||
raise VpcError(
|
||||
"member became running", 409,
|
||||
"vpc_member_requires_stopped")
|
||||
|
||||
coordinator._require_host_member_mutations_stopped = require_stopped
|
||||
coordinator._run_host_dataplane = (
|
||||
lambda *_args, **_kwargs:
|
||||
host_writes.append(_args[1]) or {})
|
||||
|
||||
asyncio.run(coordinator._reconcile_pending_hosts(
|
||||
plan, record, GATE, plan,
|
||||
host_selection=({"host-b"}, set()),
|
||||
))
|
||||
|
||||
assert acquired == [
|
||||
("host-a", INSTANCE_A),
|
||||
("host-b", INSTANCE_B),
|
||||
]
|
||||
assert checked == [
|
||||
("host-a", (INSTANCE_A,)),
|
||||
("host-b", (INSTANCE_B,)),
|
||||
]
|
||||
assert host_writes == []
|
||||
result_by_host = {host["host_id"]: host for host in record["hosts"]}
|
||||
assert result_by_host["host-b"]["state"] == "applying"
|
||||
assert result_by_host["host-b"]["last_error_code"] \
|
||||
== "vpc_member_requires_stopped"
|
||||
assert result_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
|
||||
|
||||
|
||||
def test_root_delete_without_prior_record_preflights_full_signed_vm_closure():
|
||||
fabric_plan = FabricPlan(
|
||||
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
||||
(
|
||||
FabricNode(
|
||||
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
||||
FabricNode(
|
||||
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
||||
),
|
||||
"f" * 64,
|
||||
)
|
||||
|
||||
class DeleteFabric(FakeFabric):
|
||||
@staticmethod
|
||||
def _plan_from_record(_record):
|
||||
return fabric_plan
|
||||
|
||||
async def _components(self, _plan):
|
||||
return {
|
||||
host_id: (
|
||||
SimpleNamespace(manager=object()),
|
||||
{"host_id": host_id},
|
||||
)
|
||||
for host_id in ("host-a", "host-b")
|
||||
}
|
||||
|
||||
acquired = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def vm_lock(host_id, instance_uuid):
|
||||
acquired.append((host_id, instance_uuid))
|
||||
yield
|
||||
|
||||
store = MemoryStore()
|
||||
coordinator = VpcCoordinator(
|
||||
store, DeleteFabric(hosts=("host-a", "host-b")),
|
||||
registry=object(), vm_lock=vm_lock,
|
||||
)
|
||||
coordinator._validate_dataplane_resources = lambda _plan: None
|
||||
coordinator._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
|
||||
coordinator._fabric_record = lambda _plan: {
|
||||
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
||||
}
|
||||
coordinator._fabric_peers_for_plan = lambda _plan, _record: []
|
||||
checked = []
|
||||
host_writes = []
|
||||
|
||||
def require_stopped(_profile, host_id, instance_uuids):
|
||||
checked.append((host_id, tuple(instance_uuids)))
|
||||
if host_id == "host-b":
|
||||
raise VpcError(
|
||||
"member became running", 409,
|
||||
"vpc_member_requires_stopped")
|
||||
|
||||
coordinator._require_host_member_mutations_stopped = require_stopped
|
||||
coordinator._run_host_dataplane = (
|
||||
lambda *_args, **_kwargs:
|
||||
host_writes.append(_args[1]) or {})
|
||||
target = delete_payload(
|
||||
vpc_id="vpc-never-applied",
|
||||
desired_revision=3,
|
||||
operation_id="vpc-delete-full-stop-fence-0001",
|
||||
hosts=("host-a", "host-b"),
|
||||
cidr="10.21.0.0/24",
|
||||
retiring_members=(
|
||||
{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": INSTANCE_A,
|
||||
"host_id": "host-a",
|
||||
"private_ip_cidr": "10.21.0.10/32",
|
||||
"private_mac": derive_private_mac(
|
||||
"vpc-never-applied", "member-a"),
|
||||
},
|
||||
{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"host_id": "host-b",
|
||||
"private_ip_cidr": "10.21.0.11/32",
|
||||
"private_mac": derive_private_mac(
|
||||
"vpc-never-applied", "member-b"),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(VpcError) as blocked:
|
||||
asyncio.run(coordinator.delete("vpc-never-applied", target))
|
||||
|
||||
assert blocked.value.error_code == "vpc_member_requires_stopped"
|
||||
assert acquired == [
|
||||
("host-a", INSTANCE_A),
|
||||
("host-b", INSTANCE_B),
|
||||
]
|
||||
assert checked == [
|
||||
("host-a", (INSTANCE_A,)),
|
||||
("host-b", (INSTANCE_B,)),
|
||||
]
|
||||
assert host_writes == []
|
||||
assert store.get(VPC_DELETE_RECEIPT_KIND, "vpc-never-applied") is None
|
||||
durable = _decode(store, "vpc_desired", "vpc-never-applied")
|
||||
pending = {
|
||||
host["host_id"]: host["pending_member_mutations"]
|
||||
for host in durable["hosts"]
|
||||
}
|
||||
assert pending == {
|
||||
"host-a": [INSTANCE_A],
|
||||
"host-b": [INSTANCE_B],
|
||||
}
|
||||
|
||||
|
||||
def test_root_delete_partial_retry_ignores_exact_offline_host_fence():
|
||||
vpc_id = "vpc-partial-delete"
|
||||
target = delete_payload(
|
||||
vpc_id=vpc_id,
|
||||
desired_revision=3,
|
||||
operation_id="vpc-delete-partial-fence-0001",
|
||||
hosts=("host-a", "host-b"),
|
||||
cidr="10.21.0.0/24",
|
||||
retiring_members=(
|
||||
{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": INSTANCE_A,
|
||||
"host_id": "host-a",
|
||||
"private_ip_cidr": "10.21.0.10/32",
|
||||
"private_mac": derive_private_mac(vpc_id, "member-a"),
|
||||
},
|
||||
{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"host_id": "host-b",
|
||||
"private_ip_cidr": "10.21.0.11/32",
|
||||
"private_mac": derive_private_mac(vpc_id, "member-b"),
|
||||
},
|
||||
),
|
||||
)
|
||||
plan, _manifest = parse_vpc_delete_plan(target)
|
||||
store = MemoryStore()
|
||||
first = VpcCoordinator(
|
||||
store, FakeFabric(hosts=("host-a", "host-b")), registry=object())
|
||||
first._validate_dataplane_resources = lambda _plan: None
|
||||
|
||||
async def leave_host_b_pending(
|
||||
delete_plan, record, _gate, previous_plan=None, **_kwargs):
|
||||
host_a = next(
|
||||
host for host in record["hosts"]
|
||||
if host["host_id"] == "host-a")
|
||||
confirm_delete_hosts(delete_plan, {"hosts": [host_a]})
|
||||
host_b = next(
|
||||
host for host in record["hosts"]
|
||||
if host["host_id"] == "host-b")
|
||||
host_b.update({
|
||||
"state": "uncertain",
|
||||
"transport_verified": False,
|
||||
"private_dataplane_verified": False,
|
||||
"observed_revision": 0,
|
||||
"observed_digest": None,
|
||||
"last_error_code": "vpc_host_unavailable",
|
||||
})
|
||||
|
||||
first._reconcile_dataplane = leave_host_b_pending
|
||||
with pytest.raises(VpcError) as incomplete:
|
||||
asyncio.run(first.delete(vpc_id, target))
|
||||
assert incomplete.value.error_code == "vpc_delete_incomplete"
|
||||
|
||||
durable = _decode(store, "vpc_desired", vpc_id)
|
||||
durable_hosts = {
|
||||
host["host_id"]: host for host in durable["hosts"]
|
||||
}
|
||||
assert first._confirmed_removal(
|
||||
durable_hosts["host-a"], plan.desired_revision,
|
||||
plan.desired_digest)
|
||||
assert {
|
||||
host_id: durable_hosts[host_id]["pending_member_mutations"]
|
||||
for host_id in ("host-a", "host-b")
|
||||
} == {
|
||||
"host-a": [INSTANCE_A],
|
||||
"host-b": [INSTANCE_B],
|
||||
}
|
||||
|
||||
fabric_plan = FabricPlan(
|
||||
"fabric-hk", "hk", "100.64.0.0/24", 7, 51820,
|
||||
(
|
||||
FabricNode(
|
||||
"host-a", "100.64.0.1", "192.0.2.1:51820", 51820),
|
||||
FabricNode(
|
||||
"host-b", "100.64.0.2", "192.0.2.2:51820", 51820),
|
||||
),
|
||||
"f" * 64,
|
||||
)
|
||||
component_requests = []
|
||||
|
||||
class RetryFabric(FakeFabric):
|
||||
@staticmethod
|
||||
def _plan_from_record(_record):
|
||||
return fabric_plan
|
||||
|
||||
async def _components(self, selected):
|
||||
requested = [node.host_id for node in selected.nodes]
|
||||
component_requests.append(requested)
|
||||
if "host-a" in requested:
|
||||
raise AssertionError(
|
||||
"exactly removed offline host must not be requested")
|
||||
return {
|
||||
"host-b": (
|
||||
SimpleNamespace(manager=object()),
|
||||
{"host_id": "host-b"},
|
||||
),
|
||||
}
|
||||
|
||||
acquired = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def vm_lock(host_id, instance_uuid):
|
||||
acquired.append((host_id, instance_uuid))
|
||||
yield
|
||||
|
||||
retry = VpcCoordinator(
|
||||
store, RetryFabric(hosts=("host-a", "host-b")),
|
||||
registry=object(), vm_lock=vm_lock,
|
||||
)
|
||||
retry._validate_dataplane_resources = lambda _plan: None
|
||||
retry._mark_lifecycle_conflicts = lambda *_args, **_kwargs: False
|
||||
retry._fabric_record = lambda _plan: {
|
||||
"public_keys": {"host-a": "a" * 44, "host-b": "b" * 44},
|
||||
}
|
||||
retry._fabric_peers_for_plan = lambda _plan, _record: []
|
||||
stopped = []
|
||||
host_runs = []
|
||||
|
||||
def require_stopped(_profile, host_id, instance_uuids):
|
||||
stopped.append((host_id, tuple(instance_uuids)))
|
||||
|
||||
def confirm_host(_plan, host_id, *_args, **_kwargs):
|
||||
host_runs.append(host_id)
|
||||
assert host_id == "host-b"
|
||||
return {
|
||||
"state": "active",
|
||||
"removed": True,
|
||||
"transport_verified": True,
|
||||
"private_dataplane_verified": True,
|
||||
"config_verified": True,
|
||||
"config_observed_revision": plan.desired_revision,
|
||||
"config_observed_digest": plan.desired_digest,
|
||||
"member_nics": [],
|
||||
"routes": [],
|
||||
"wg_allowed_ips": [],
|
||||
"proxy_ips": [],
|
||||
"bridge": None,
|
||||
"bridge_owner": None,
|
||||
"nft_table": None,
|
||||
"anti_spoof": False,
|
||||
"rp_filter": None,
|
||||
"proxy_arp": None,
|
||||
"proxy_arp_pvlan": None,
|
||||
"readback": "confirmed",
|
||||
"rollback": "none",
|
||||
"reason_code": None,
|
||||
}
|
||||
|
||||
retry._require_host_member_mutations_stopped = require_stopped
|
||||
retry._run_host_dataplane = confirm_host
|
||||
result = asyncio.run(retry.delete(vpc_id, target))
|
||||
|
||||
assert component_requests == [["host-b"]]
|
||||
assert acquired == [("host-b", INSTANCE_B)]
|
||||
assert stopped == [("host-b", (INSTANCE_B,))]
|
||||
assert host_runs == ["host-b"]
|
||||
assert result["deleted"] is True
|
||||
assert result["removed_host_ids"] == ["host-a", "host-b"]
|
||||
assert result["readback"] == "confirmed"
|
||||
receipt = _decode(store, VPC_DELETE_RECEIPT_KIND, vpc_id)
|
||||
assert receipt["operation_id"] == plan.operation_id
|
||||
assert receipt["desired_digest"] == plan.desired_digest
|
||||
|
||||
|
||||
def test_legacy_exact_host_migrates_empty_pending_without_host_write():
|
||||
store = MemoryStore()
|
||||
coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",)))
|
||||
plan = parse_vpc_plan(payload())
|
||||
record = coordinator._desired_record(plan, GATE, state="applying")
|
||||
record["hosts"] = [_exact_host(plan, "host-a")]
|
||||
coordinator._finish(record, GATE)
|
||||
coordinator._save("vpc_desired", plan.vpc_id, record)
|
||||
coordinator._save_children(record)
|
||||
host_writes = []
|
||||
|
||||
async def unexpected_host_write(*_args, **_kwargs):
|
||||
host_writes.append(True)
|
||||
|
||||
coordinator._reconcile_dataplane = unexpected_host_write
|
||||
asyncio.run(coordinator.reconcile(plan.vpc_id))
|
||||
|
||||
durable = _decode(store, "vpc_desired", plan.vpc_id)
|
||||
durable_host = _decode(
|
||||
store, "vpc_host", f"{plan.vpc_id}.host-a")
|
||||
assert host_writes == []
|
||||
assert durable["hosts"][0]["pending_member_mutations"] == []
|
||||
assert durable_host["pending_member_mutations"] == []
|
||||
|
||||
|
||||
def test_explicit_empty_pending_fence_is_not_rederived_from_current_members():
|
||||
coordinator = VpcCoordinator(
|
||||
MemoryStore(), FakeFabric(hosts=("host-a",)))
|
||||
plan = parse_vpc_plan(payload())
|
||||
record = coordinator._desired_record(plan, GATE, state="applying")
|
||||
host = record["hosts"][0]
|
||||
host["pending_member_mutations"] = []
|
||||
host["config_verified"] = False
|
||||
host["config_observed_revision"] = 0
|
||||
host["config_observed_digest"] = None
|
||||
|
||||
assert coordinator._migrate_legacy_pending_member_mutations(
|
||||
record, plan) is False
|
||||
assert host["pending_member_mutations"] == []
|
||||
|
||||
|
||||
def test_legacy_incomplete_current_and_retiring_hosts_derive_stop_closure():
|
||||
coordinator = VpcCoordinator(
|
||||
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
|
||||
plan = parse_vpc_plan(_two_host_payload())
|
||||
|
||||
current = _exact_host(plan, "host-a")
|
||||
current.update({
|
||||
"state": "applying",
|
||||
"config_verified": False,
|
||||
"host_config_reusable": False,
|
||||
"config_observed_revision": 0,
|
||||
"config_observed_digest": None,
|
||||
"member_nics": [],
|
||||
})
|
||||
current_record = coordinator._desired_record(
|
||||
plan, GATE, state="applying")
|
||||
current_record["hosts"] = [
|
||||
current if host["host_id"] == "host-a" else host
|
||||
for host in current_record["hosts"]
|
||||
]
|
||||
assert coordinator._migrate_legacy_pending_member_mutations(
|
||||
current_record, plan) is True
|
||||
current_by_host = {
|
||||
host["host_id"]: host for host in current_record["hosts"]}
|
||||
assert current_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
|
||||
|
||||
retiring = {
|
||||
"host_id": "host-a",
|
||||
"vpc_id": plan.vpc_id,
|
||||
"member_ids": ["member-a"],
|
||||
"retiring": True,
|
||||
"retiring_members": [{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": INSTANCE_A,
|
||||
"host_id": "host-a",
|
||||
"private_ip_cidr": "10.20.0.10/32",
|
||||
"subnet_id": "subnet-a",
|
||||
"private_mac": derive_private_mac(plan.vpc_id, "member-a"),
|
||||
}],
|
||||
}
|
||||
host_b_only = parse_vpc_plan(payload(
|
||||
hosts=[{"host_id": "host-b", "members": [{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"private_ip_cidr": "10.20.0.11/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}]}],
|
||||
))
|
||||
retiring_record = coordinator._desired_record(
|
||||
host_b_only, GATE, state="applying")
|
||||
retiring_record["hosts"].append(retiring)
|
||||
assert coordinator._migrate_legacy_pending_member_mutations(
|
||||
retiring_record, host_b_only) is True
|
||||
retiring_by_host = {
|
||||
host["host_id"]: host for host in retiring_record["hosts"]}
|
||||
assert retiring_by_host["host-a"]["pending_member_mutations"] == [INSTANCE_A]
|
||||
|
||||
|
||||
def test_legacy_retiring_host_with_malformed_identity_fails_closed():
|
||||
coordinator = VpcCoordinator(
|
||||
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
|
||||
plan = parse_vpc_plan(payload(
|
||||
hosts=[{"host_id": "host-b", "members": [{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"private_ip_cidr": "10.20.0.11/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}]}],
|
||||
))
|
||||
malformed = {
|
||||
"host_id": "host-a",
|
||||
"vpc_id": plan.vpc_id,
|
||||
"member_ids": ["member-a"],
|
||||
"retiring": True,
|
||||
"retiring_members": [{
|
||||
"member_id": "member-a",
|
||||
"instance_uuid": "not-a-domain-uuid",
|
||||
"host_id": "host-a",
|
||||
"private_ip_cidr": "10.20.0.10/32",
|
||||
"subnet_id": "subnet-a",
|
||||
"private_mac": derive_private_mac(plan.vpc_id, "member-a"),
|
||||
}],
|
||||
}
|
||||
|
||||
record = coordinator._desired_record(plan, GATE, state="applying")
|
||||
record["hosts"].append(malformed)
|
||||
|
||||
with pytest.raises(VpcError) as error:
|
||||
coordinator._migrate_legacy_pending_member_mutations(record, plan)
|
||||
|
||||
assert error.value.error_code == "vpc_record_corrupt"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retiring_members", [None, []])
|
||||
def test_legacy_retiring_host_requires_complete_member_closure(
|
||||
retiring_members):
|
||||
coordinator = VpcCoordinator(
|
||||
MemoryStore(), FakeFabric(hosts=("host-a", "host-b")))
|
||||
plan = parse_vpc_plan(payload(
|
||||
hosts=[{"host_id": "host-b", "members": [{
|
||||
"member_id": "member-b",
|
||||
"instance_uuid": INSTANCE_B,
|
||||
"private_ip_cidr": "10.20.0.11/32",
|
||||
"subnet_id": "subnet-a",
|
||||
}]}],
|
||||
))
|
||||
retiring = {
|
||||
"host_id": "host-a",
|
||||
"vpc_id": plan.vpc_id,
|
||||
"member_ids": ["member-a"],
|
||||
"retiring": True,
|
||||
}
|
||||
if retiring_members is not None:
|
||||
retiring["retiring_members"] = retiring_members
|
||||
|
||||
record = coordinator._desired_record(plan, GATE, state="applying")
|
||||
record["hosts"].append(retiring)
|
||||
|
||||
with pytest.raises(VpcError) as error:
|
||||
coordinator._migrate_legacy_pending_member_mutations(record, plan)
|
||||
|
||||
assert error.value.error_code == "vpc_record_corrupt"
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue