You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_vpc_operation_identity.py

1752 lines
74 KiB
Python

"""VPC claims bind exact immutable target identity before host admission."""
import asyncio
import json
import threading
import pytest
from virsh_panel.coordination import (
CoordinationAdmissionRejected, CoordinationClaimRejected, CoordinationError,
DurableCoordinator, current_execution, execution_context,
mark_execution_unknown,
)
from virsh_panel.vpc import (
VpcCoordinator, VpcError, parse_vpc_delete_plan, parse_vpc_plan,
)
from virsh_panel.vpc_dataplane import derive_private_mac
from test_durable_fabric_jobs import FabricStore
from test_engine_boundary import SUPER, cloud_engine_request
from test_engine_coordination import local_backends, request, running_engine
from test_vpc import (
FakeFabric, INSTANCE_A, INSTANCE_B, confirm_delete_hosts, delete_payload,
payload,
)
from test_vpc_admission_coordination import vpc_payload
OPERATION_ID = "a" * 32
def identity(plan):
return {"job_kind": "vpc_operation", "vpc_id": plan.vpc_id,
"desired_revision": plan.desired_revision,
"desired_digest": plan.desired_digest}
def assert_retained_claim_family(rows, plan, *, root_kind, member_uuids):
expected_kinds = {
(): root_kind,
("catalog:network",): "resource",
**{
(f"vm:host-a:{instance_uuid}",): "resource"
for instance_uuid in member_uuids
},
}
by_resources = {
tuple(row["resources"]): row
for row in rows.values()
}
assert len(rows) == len(expected_kinds)
assert set(by_resources) == set(expected_kinds)
for resources, expected_kind in expected_kinds.items():
row = by_resources[resources]
assert row["kind"] == expected_kind
assert row["state"] == "uncertain"
assert row["operation_id"] == plan.operation_id
assert row["metadata"] == identity(plan)
async def explicit_request(engine, method, target, value=None):
actor = {**SUPER, "vpc_reconcile_explicit": True}
body = b"" if value is None else json.dumps(value).encode()
return await cloud_engine_request(
engine, method, target, body, actor,
headers={"Content-Type": "application/json"},
)
def retained_root(coordinator, plan, *, kind="background"):
claim = coordinator.claim(
operation_id=plan.operation_id, kind=kind, metadata=identity(plan))
assert coordinator.retain(claim)
return coordinator.snapshot()[claim.token]
def coordination_document(store):
return json.loads(store.get("coordination", "admission-v1"))
def retiring_members(value):
plan = parse_vpc_plan(value)
return tuple({
"member_id": member.member_id,
"instance_uuid": member.instance_uuid,
"host_id": member.host_id,
"private_ip_cidr": f"{member.private_ip}/32",
"private_mac": (member.private_mac
or derive_private_mac(plan.vpc_id, member.member_id)),
} for member in plan.members)
async def seed(engine, store, *, empty=False):
target = payload(hosts=[], members=[], operation_id=OPERATION_ID) if empty else {
**vpc_payload(), "operation_id": OPERATION_ID}
vpc = VpcCoordinator(store, FakeFabric(hosts=("host-a",)),
catalog_lock=engine._network_catalog_mutation_lock,
vm_lock=engine._target_lock, coordinator=engine.coordinator)
await vpc.apply(target)
assert engine.coordinator.snapshot() == {}
engine.vpc = vpc
return vpc, target
@pytest.mark.parametrize("old_kind", ["background", "request"])
def test_explicit_http_apply_atomically_takes_over_exact_retained_root(
tmp_path, monkeypatch, old_kind):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
old = retained_root(engine.coordinator, plan, kind=old_kind)
seen = []
async def host(*_args, **_kwargs):
rows = list(engine.coordinator.snapshot().values())
seen.extend(rows)
assert rows
assert all(row["operation_id"] == plan.operation_id
and row["metadata"] == identity(plan)
and "explicit_vpc_reconcile" not in row["metadata"]
for row in rows)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 200, result
assert seen
assert engine.coordinator.snapshot() == {}
document = coordination_document(store)
receipts = [item for item in document["resolutions"]
if item.get("kind") == "resolve_and_bind_v1"]
assert len(receipts) == 1
receipt = receipts[0]
assert receipt["token"] == old["token"]
assert receipt["operation_id"] == plan.operation_id
assert receipt["replacement_token"] != old["token"]
assert set(receipt) == {
"kind", "token", "operation_id", "resolved_by",
"resolved_at", "evidence", "source_claim_sha256",
"replacement_token", "replacement_identity_sha256",
}
asyncio.run(run())
def test_explicit_http_apply_without_retained_root_consumes_marker(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
seen = []
async def host(*_args, **_kwargs):
seen.extend(engine.coordinator.snapshot().values())
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 200, result
assert seen
assert all(row["metadata"] == identity(plan)
and "explicit_vpc_reconcile" not in row["metadata"]
for row in seen)
assert engine.coordinator.snapshot() == {}
assert coordination_document(store)["resolutions"] == []
asyncio.run(run())
def test_http_apply_without_explicit_marker_keeps_exact_retained_root(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
old = retained_root(
engine.coordinator, parse_vpc_plan(target))
before = engine.coordinator.snapshot()
async def forbidden(*_args, **_kwargs):
pytest.fail("unmarked retry reached host dispatch")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert engine.coordinator.snapshot() == before
assert old["token"] in before
assert coordination_document(store)["resolutions"] == []
asyncio.run(run())
def test_explicit_http_delete_takes_over_only_durable_delete_target(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, applied = await seed(engine, store)
target = delete_payload(
operation_id="b" * 32,
retiring_members=retiring_members(applied),
)
plan = parse_vpc_delete_plan(target)[0]
release = engine.coordinator.release
failed_root_release = False
async def confirmed(plan_value, record, *_args, **_kwargs):
confirm_delete_hosts(plan_value, record)
def lose_background_release(claim):
nonlocal failed_root_release
kind = (claim.get("kind") if isinstance(claim, dict)
else claim.kind)
if kind == "background" and not failed_root_release:
failed_root_release = True
raise RuntimeError("lost background release response")
return release(claim)
monkeypatch.setattr(vpc, "_reconcile_dataplane", confirmed)
monkeypatch.setattr(
engine.coordinator, "release", lose_background_release)
with pytest.raises(RuntimeError):
await vpc.delete("vpc-hk-1", target)
assert failed_root_release
rows = engine.coordinator.snapshot()
assert len(rows) == 1
old = next(iter(rows.values()))
assert old["state"] == "uncertain"
assert old["operation_id"] == plan.operation_id
assert old["metadata"] == identity(plan)
assert store.get("vpc_desired", "vpc-hk-1") is None
monkeypatch.setattr(engine.coordinator, "release", release)
monkeypatch.setattr(
vpc, "_reconcile_dataplane",
lambda *_args, **_kwargs: pytest.fail(
"confirmed delete receipt retried host dispatch"))
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
assert status == 200, result
assert result["deleted"] is True
assert engine.coordinator.snapshot() == {}
receipts = coordination_document(store)["resolutions"]
assert any(item.get("kind") == "resolve_and_bind_v1"
and item.get("token") == old["token"]
for item in receipts)
asyncio.run(run())
def test_http_delete_without_explicit_marker_keeps_retained_delete_root(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, applied = await seed(engine, store)
target = delete_payload(
operation_id="c" * 32,
retiring_members=retiring_members(applied),
)
plan = parse_vpc_delete_plan(target)[0]
release = engine.coordinator.release
failed_root_release = False
async def confirmed(plan_value, record, *_args, **_kwargs):
confirm_delete_hosts(plan_value, record)
def lose_background_release(claim):
nonlocal failed_root_release
kind = (claim.get("kind") if isinstance(claim, dict)
else claim.kind)
if kind == "background" and not failed_root_release:
failed_root_release = True
raise RuntimeError("lost background release response")
return release(claim)
monkeypatch.setattr(vpc, "_reconcile_dataplane", confirmed)
monkeypatch.setattr(
engine.coordinator, "release", lose_background_release)
with pytest.raises(RuntimeError):
await vpc.delete("vpc-hk-1", target)
assert failed_root_release
monkeypatch.setattr(engine.coordinator, "release", release)
before = engine.coordinator.snapshot()
async def forbidden(*_args, **_kwargs):
pytest.fail("unmarked delete retry reached host dispatch")
monkeypatch.setattr(vpc, "_reconcile_dataplane", forbidden)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert engine.coordinator.snapshot() == before
assert coordination_document(store)["resolutions"] == []
asyncio.run(run())
def test_explicit_delete_does_not_take_over_root_with_residual_child(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, applied = await seed(engine, store)
target = delete_payload(
operation_id="e" * 32,
retiring_members=retiring_members(applied),
)
async def interrupted(*_args, **_kwargs):
raise CoordinationError("coordination_unavailable", 503)
monkeypatch.setattr(vpc, "_reconcile_dataplane", interrupted)
with pytest.raises(CoordinationError):
await vpc.delete("vpc-hk-1", target)
before = engine.coordinator.snapshot()
assert len(before) == 2
execution_ids = {row["execution_id"] for row in before.values()}
assert len(execution_ids) == 1
assert any(row["kind"] == "background" for row in before.values())
assert any(row["kind"] == "resource" for row in before.values())
async def forbidden(*_args, **_kwargs):
pytest.fail("residual-child delete reached host dispatch")
monkeypatch.setattr(vpc, "_reconcile_dataplane", forbidden)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert engine.coordinator.snapshot() == before
assert coordination_document(store)["resolutions"] == []
asyncio.run(run())
def test_explicit_apply_revalidates_durable_target_before_handoff(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, stale = await seed(engine, store)
stale_plan = parse_vpc_plan(stale)
old = retained_root(engine.coordinator, stale_plan)
newer = {
**stale,
"desired_revision": stale_plan.desired_revision + 1,
"operation_id": "f" * 32,
}
await vpc.apply(newer)
before = engine.coordinator.snapshot()
assert set(before) == {old["token"]}
async def forbidden(*_args, **_kwargs):
pytest.fail("stale retained target reached host dispatch")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", stale)
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert engine.coordinator.snapshot() == before
assert coordination_document(store)["resolutions"] == []
asyncio.run(run())
@pytest.mark.parametrize("fault", ["before", "after", "false_after"])
def test_explicit_handoff_cas_fault_is_read_back_fenced_and_retryable(
tmp_path, monkeypatch, fault):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
old = retained_root(engine.coordinator, plan)
compare = engine.coordinator.store.compare_and_set
triggered = False
host_calls = 0
def fail_handoff(kind, key, expected, value):
nonlocal triggered
desired = json.loads(value)
handoff = any(
item.get("kind") == "resolve_and_bind_v1"
and item.get("token") == old["token"]
for item in desired.get("resolutions", []))
if handoff and not triggered:
triggered = True
if fault == "before":
raise RuntimeError("handoff did not commit")
assert compare(kind, key, expected, value)
if fault == "after":
raise RuntimeError("handoff response lost")
return False
return compare(kind, key, expected, value)
async def host(*_args, **_kwargs):
nonlocal host_calls
host_calls += 1
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", fail_handoff)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 503, result
assert result["error_code"] in {
"coordination_uncertain", "coordination_unavailable"}
assert triggered and host_calls == 0
rows = engine.coordinator.snapshot()
assert old["token"] not in rows
assert len(rows) == 1
root = next(iter(rows.values()))
assert root["state"] == "uncertain"
assert root["kind"] == "request"
assert root["resources"] == []
assert root["operation_id"] == plan.operation_id
assert root["metadata"] == identity(plan)
receipts = [item for item in coordination_document(store)["resolutions"]
if item.get("kind") == "resolve_and_bind_v1"]
assert len(receipts) == 1
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", compare)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 200, result
assert host_calls == 1
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
@pytest.mark.parametrize("fault", ["before", "after", "false_after"])
def test_marker_only_bind_cas_fault_strips_capability_and_is_retryable(
tmp_path, monkeypatch, fault):
async def run():
async with running_engine(tmp_path) as (engine, _store, _services):
vpc, target = await seed(engine, _store)
plan = parse_vpc_plan(target)
compare = engine.coordinator.store.compare_and_set
triggered = False
host_calls = 0
def fail_bind(kind, key, expected, value):
nonlocal triggered
before = json.loads(expected).get("claims", {})
after = json.loads(value).get("claims", {})
bound = any(
token in before
and before[token].get("operation_id") is None
and before[token].get("metadata")
== {"explicit_vpc_reconcile": True}
and row.get("operation_id") == plan.operation_id
and row.get("metadata") == identity(plan)
for token, row in after.items())
if bound and not triggered:
triggered = True
if fault == "before":
raise RuntimeError("bind did not commit")
assert compare(kind, key, expected, value)
if fault == "after":
raise RuntimeError("bind response lost")
return False
return compare(kind, key, expected, value)
async def host(*_args, **_kwargs):
nonlocal host_calls
host_calls += 1
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", fail_bind)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 503, result
assert triggered and host_calls == 0
rows = engine.coordinator.snapshot()
assert len(rows) == 1
root = next(iter(rows.values()))
assert root["state"] == "uncertain"
assert root["operation_id"] == plan.operation_id
assert root["metadata"] == identity(plan)
assert "explicit_vpc_reconcile" not in root["metadata"]
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", compare)
status, result = await explicit_request(
engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 200, result
assert host_calls == 1
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
@pytest.mark.parametrize("phase", ["handoff", "bind"])
def test_explicit_admission_cancellation_fences_exact_bound_root(
tmp_path, monkeypatch, phase):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
old = (retained_root(engine.coordinator, plan)
if phase == "handoff" else None)
method_name = ("resolve_and_bind" if phase == "handoff"
else "bind_explicit_operation")
original = getattr(engine.coordinator, method_name)
entered = threading.Event()
resume = threading.Event()
def commit_then_block(*args, **kwargs):
result = original(*args, **kwargs)
entered.set()
assert resume.wait(5), "test did not release admission CAS"
return result
monkeypatch.setattr(engine.coordinator, method_name, commit_then_block)
parent = engine.coordinator.claim(
kind="request", metadata={"explicit_vpc_reconcile": True})
async def apply_in_parent():
with execution_context(engine.coordinator, parent):
return await vpc.apply(target)
task = asyncio.create_task(apply_in_parent())
try:
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
finally:
resume.set()
with pytest.raises(asyncio.CancelledError):
await task
rows = engine.coordinator.snapshot()
if old is not None:
assert old["token"] not in rows
assert len(rows) == 1
root = next(iter(rows.values()))
assert root["token"] == parent.token
assert root["state"] == "uncertain"
assert root["operation_id"] == plan.operation_id
assert root["metadata"] == identity(plan)
asyncio.run(run())
@pytest.mark.parametrize("child_execution", ["old", "replacement"])
def test_atomic_handoff_rejects_any_child_of_either_execution(
tmp_path, child_execution):
store = FabricStore()
coordinator = DurableCoordinator(store, owner_id="d" * 64)
target = payload(operation_id=OPERATION_ID)
plan = parse_vpc_plan(target)
old_claim = coordinator.claim(
operation_id=plan.operation_id, kind="background",
metadata=identity(plan))
parent = coordinator.claim(
kind="request", metadata={"explicit_vpc_reconcile": True})
execution_id = (old_claim.execution_id if child_execution == "old"
else parent.execution_id)
child = coordinator.claim(
operation_id=(plan.operation_id if child_execution == "old" else None),
resources=(f"test:{child_execution}-execution-child",),
execution_id=execution_id, kind="resource",
metadata=(identity(plan) if child_execution == "old"
else {"explicit_vpc_reconcile": True}))
assert coordinator.retain(old_claim)
before = coordinator.snapshot()
old = before[old_claim.token]
with pytest.raises(CoordinationClaimRejected):
coordinator.resolve_and_bind(
old, parent, "exact retained root evidence",
operation_id=plan.operation_id, metadata=identity(plan))
assert coordinator.snapshot() == before
assert child.token in before
assert coordination_document(store)["resolutions"] == []
@pytest.mark.parametrize("invalid", ["kind", "resources", "metadata"])
def test_atomic_handoff_rejects_noncanonical_retained_root(invalid):
store = FabricStore()
coordinator = DurableCoordinator(store, owner_id="d" * 64)
plan = parse_vpc_plan(payload(operation_id=OPERATION_ID))
kwargs = {
"operation_id": plan.operation_id,
"kind": "background",
"metadata": identity(plan),
}
if invalid == "kind":
kwargs["kind"] = "resource"
elif invalid == "resources":
kwargs["resources"] = ("test:noncanonical-root",)
else:
kwargs["metadata"] = {**identity(plan), "desired_revision": 999}
old_claim = coordinator.claim(**kwargs)
assert coordinator.retain(old_claim)
parent = coordinator.claim(
kind="request", metadata={"explicit_vpc_reconcile": True})
before = coordinator.snapshot()
with pytest.raises(CoordinationClaimRejected):
coordinator.resolve_and_bind(
before[old_claim.token], parent,
"exact retained root evidence",
operation_id=plan.operation_id, metadata=identity(plan))
assert coordinator.snapshot() == before
assert coordination_document(store)["resolutions"] == []
def test_handoff_readback_rejects_mismatched_related_receipt():
store = FabricStore()
coordinator = DurableCoordinator(store, owner_id="d" * 64)
plan = parse_vpc_plan(payload(operation_id=OPERATION_ID))
old_claim = coordinator.claim(
operation_id=plan.operation_id, kind="background",
metadata=identity(plan))
assert coordinator.retain(old_claim)
old = coordinator.snapshot()[old_claim.token]
parent = coordinator.claim(
kind="request", metadata={"explicit_vpc_reconcile": True})
evidence = "exact retained root evidence"
coordinator.resolve_and_bind(
old, parent, evidence,
operation_id=plan.operation_id, metadata=identity(plan))
document = coordination_document(store)
document["resolutions"][-1]["evidence"] = "different evidence"
store.set("coordination", "admission-v1", json.dumps(document))
outcome, current = coordinator.classify_resolve_and_bind(
old, parent, evidence,
operation_id=plan.operation_id, metadata=identity(plan))
assert outcome == "mixed"
assert current["token"] == parent.token
@pytest.mark.parametrize("entry", ["apply", "reconcile"])
@pytest.mark.parametrize("unknown", [False, True])
def test_http_vpc_claims_use_signed_target_and_never_generic_vm_receipt(
tmp_path, monkeypatch, entry, unknown):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
seen = []
# An unrelated generic VM receipt must not settle VPC work just
# because it happens to share the opaque operation identifier.
store.set("operation", OPERATION_ID, json.dumps({"mutation_started": False}))
async def host(*_args, **_kwargs):
rows = list(engine.coordinator.snapshot().values())
seen.append(rows)
assert len(rows) == 4 # request, catalog, both member VM locks
assert {row["operation_id"] for row in rows} == {OPERATION_ID}
assert all(row["metadata"] == identity(plan) for row in rows)
assert current_execution().claim.operation_id == OPERATION_ID
if unknown:
raise CoordinationError("coordination_uncertain", 503)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await request(engine, "POST",
"/internal/v1/vpcs/apply" if entry == "apply" else "/internal/v1/vpcs/vpc-hk-1/reconcile",
target if entry == "apply" else {}, operation_id="actor-id-is-not-the-vpc-target")
assert seen
assert status == (503 if unknown else 200), result
rows = list(engine.coordinator.snapshot().values())
if unknown:
assert len(rows) == 4
assert all(row["state"] == "uncertain" and row["operation_id"] == OPERATION_ID
and row["metadata"] == identity(plan) for row in rows)
else:
assert rows == []
asyncio.run(run())
@pytest.mark.parametrize("unknown", [False, True])
def test_periodic_vpc_claims_are_named_and_unknown_retry_keeps_original_fence(
tmp_path, monkeypatch, unknown):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
seen = []
async def host(*_args, **_kwargs):
rows = list(engine.coordinator.snapshot().values())
seen.append(rows)
assert len(rows) == 4
assert any(row["kind"] == "background" for row in rows)
assert all(row["operation_id"] == OPERATION_ID
and row["metadata"] == identity(plan) for row in rows)
if unknown:
raise CoordinationError("coordination_uncertain", 503)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
if unknown:
with pytest.raises(CoordinationError):
await vpc.reconcile("vpc-hk-1")
before = engine.coordinator.snapshot()
assert len(before) == 4
assert all(row["state"] == "uncertain" for row in before.values())
with pytest.raises(CoordinationAdmissionRejected):
await vpc.reconcile("vpc-hk-1")
await vpc.reconcile_all()
assert engine.coordinator.snapshot() == before
assert len(seen) == 1
else:
await vpc.reconcile("vpc-hk-1")
assert seen and engine.coordinator.snapshot() == {}
asyncio.run(run())
@pytest.mark.parametrize("state", ["active", "uncertain"])
@pytest.mark.parametrize("entry", ["http", "periodic"])
def test_same_operation_admission_rejection_does_not_create_anonymous_uncertainty(
tmp_path, monkeypatch, state, entry):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
owner = DurableCoordinator(store)
blocker = owner.claim(operation_id=OPERATION_ID, kind="resource")
if state == "uncertain": owner.retain(blocker)
before = owner.snapshot()
async def forbidden(*_args, **_kwargs):
pytest.fail("rejected operation reached host dispatch")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
if entry == "http":
status, result = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == (503 if state == "uncertain" else 409), result
else:
with pytest.raises(CoordinationAdmissionRejected):
await vpc.reconcile("vpc-hk-1")
assert owner.snapshot() == before
asyncio.run(run())
def test_http_delete_claim_binds_the_delete_target_before_receipt_write(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, _ = await seed(engine, store, empty=True)
target = delete_payload(hosts=(), operation_id="b" * 32)
observed = []
original = vpc._save
def save(kind, key, value):
if kind == "vpc_deleted":
rows = list(engine.coordinator.snapshot().values())
observed.extend(rows)
assert len(rows) == 2
assert all(row["operation_id"] == target["operation_id"] for row in rows)
assert all(row["metadata"]["vpc_id"] == "vpc-hk-1"
and row["metadata"]["desired_revision"] == 2 for row in rows)
return original(kind, key, value)
monkeypatch.setattr(vpc, "_save", save)
status, result = await request(engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
assert status == 200 and result["deleted"] is True, result
assert observed and engine.coordinator.snapshot() == {}
asyncio.run(run())
@pytest.mark.parametrize("entry", ["http", "periodic"])
def test_legacy_anonymous_catalog_claim_is_preserved_without_new_uncertainty(
tmp_path, monkeypatch, entry):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
owner = DurableCoordinator(store)
old = owner.claim(resources=("catalog:network",), kind="resource")
owner.retain(old)
before = owner.snapshot()
async def forbidden(*_args, **_kwargs):
pytest.fail("legacy unknown owner was bypassed")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
if entry == "http":
status, result = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
assert status == 503 and result["error_code"] == "coordination_uncertain", result
else:
with pytest.raises(CoordinationAdmissionRejected):
await vpc.reconcile("vpc-hk-1")
assert owner.snapshot() == before
asyncio.run(run())
@pytest.mark.parametrize("fault", ["missing_id", "invalid_id", "digest", "revision"])
def test_invalid_signed_vpc_envelope_never_binds_or_dispatches(tmp_path, monkeypatch, fault):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
target = dict(target)
if fault == "missing_id": target.pop("operation_id")
elif fault == "invalid_id": target["operation_id"] = "../unsafe"
elif fault == "digest": target["desired_digest"] = "0" * 64
else: target["desired_revision"] = 0
bound = []
bind = engine.coordinator.bind_operation
def track(*args, **kwargs):
bound.append(True)
return bind(*args, **kwargs)
monkeypatch.setattr(engine.coordinator, "bind_operation", track)
status, _ = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
assert status in {400, 409, 422}
assert not bound and engine.coordinator.snapshot() == {}
asyncio.run(run())
def test_reconcile_identity_change_before_catalog_admission_is_rejected(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
bind = engine.coordinator.bind_operation
def supersede(*args, **kwargs):
result = bind(*args, **kwargs)
record = json.loads(store.get("vpc_desired", "vpc-hk-1"))
record["operation_id"] = "b" * 32
store.set("vpc_desired", "vpc-hk-1", json.dumps(record))
return result
async def forbidden(*_args, **_kwargs):
pytest.fail("stale bound identity reached host dispatch")
monkeypatch.setattr(engine.coordinator, "bind_operation", supersede)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
status, result = await request(engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 409 and result["error_code"] == "vpc_revision_conflict", result
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
def test_bind_operation_is_immutable_and_keeps_ownership_and_extra_metadata():
owner = DurableCoordinator(FabricStore())
claim = owner.claim(kind="request", metadata={"audit": "preserved"})
metadata = identity(parse_vpc_plan(payload(operation_id=OPERATION_ID)))
bound = owner.bind_operation(claim, OPERATION_ID, metadata)
assert (bound.token, bound.execution_id, bound.owner_id) == (claim.token, claim.execution_id, claim.owner_id)
assert bound.metadata == {"audit": "preserved", **metadata}
assert owner.bind_operation(bound, OPERATION_ID, metadata) == bound
before = owner.snapshot()
for operation, changed in [("b" * 32, metadata), (OPERATION_ID, {**metadata, "vpc_id": "another-vpc"})]:
with pytest.raises(CoordinationClaimRejected):
owner.bind_operation(bound, operation, changed)
assert owner.snapshot() == before
assert owner.release(claim) and owner.snapshot() == {}
def test_lost_bind_response_preserves_committed_operation_identity():
class LostReply(FabricStore):
fail = False
def compare_and_set(self, kind, key, expected, value):
result = super().compare_and_set(kind, key, expected, value)
if result and self.fail:
self.fail = False
raise RuntimeError("committed bind reply lost")
return result
store = LostReply()
owner = DurableCoordinator(store)
claim = owner.claim(kind="request")
store.fail = True
with execution_context(owner, claim):
with pytest.raises(CoordinationError):
owner.bind_operation(claim, OPERATION_ID, {"job_kind": "vpc_operation"})
assert current_execution().execution_unknown.is_set()
assert owner.snapshot()[claim.token]["operation_id"] == OPERATION_ID
@pytest.mark.parametrize("entry", ["http", "periodic"])
def test_legacy_anonymous_empty_claim_is_not_borrowed_or_a_vpc_blocker(
tmp_path, entry):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, _target = await seed(engine, store)
legacy_owner = DurableCoordinator(store)
legacy = legacy_owner.claim(kind="request")
assert legacy_owner.retain(legacy)
before = legacy_owner.snapshot()
row = before[legacy.token]
assert row["state"] == "uncertain"
assert row["operation_id"] is None
assert row["resources"] == []
assert row["metadata"] == {}
if entry == "http":
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 200, result
else:
await vpc.reconcile("vpc-hk-1")
# The legacy row has no target identity or resource fence, so it
# cannot safely be borrowed by this operation. It also does not
# intersect the exact VPC operation/catalog claims admitted above.
assert legacy_owner.snapshot() == before
# It still blocks process-wide maintenance until an operator can
# supply explicit evidence to DurableCoordinator.resolve().
with pytest.raises(CoordinationClaimRejected) as blocked:
legacy_owner.claim(maintenance=True, kind="update")
assert blocked.value.error_code == "coordination_uncertain"
asyncio.run(run())
def test_periodic_cancel_after_claim_commit_retains_exact_owner(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
committed = threading.Event()
reply = threading.Event()
claim = engine.coordinator.claim
def delayed_reply(*args, **kwargs):
result = claim(*args, **kwargs)
if kwargs.get("kind") == "background":
committed.set()
assert reply.wait(5), "test did not release committed claim response"
return result
async def forbidden(*_args, **_kwargs):
pytest.fail("cancelled admission dispatched host work")
monkeypatch.setattr(engine.coordinator, "claim", delayed_reply)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
try:
assert await asyncio.to_thread(committed.wait, 5)
task.cancel()
finally:
reply.set()
with pytest.raises(asyncio.CancelledError):
await task
before = engine.coordinator.snapshot()
assert len(before) == 1
row = next(iter(before.values()))
assert row["state"] == "uncertain" and row["kind"] == "background"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(parse_vpc_plan(target))
with pytest.raises(CoordinationAdmissionRejected):
await vpc.reconcile("vpc-hk-1")
assert engine.coordinator.snapshot() == before
asyncio.run(run())
def test_periodic_cancel_during_lost_claim_recovery_fences_exact_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
claim = engine.coordinator.claim
snapshot = engine.coordinator.snapshot
entered = threading.Event()
resume = threading.Event()
blocked = False
def commit_then_raise(*args, **kwargs):
result = claim(*args, **kwargs)
if kwargs.get("kind") == "background":
raise RuntimeError("committed background claim reply lost")
return result
def block_recovery_snapshot():
nonlocal blocked
rows = snapshot()
if (not blocked and len(rows) == 1
and next(iter(rows.values())).get("kind") == "background"):
blocked = True
entered.set()
assert resume.wait(5), "test did not release recovery snapshot"
return rows
async def forbidden(*_args, **_kwargs):
pytest.fail("unknown background admission dispatched host work")
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
monkeypatch.setattr(engine.coordinator, "snapshot", block_recovery_snapshot)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
try:
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
finally:
resume.set()
with pytest.raises(asyncio.CancelledError):
await task
rows = snapshot()
assert len(rows) == 1
row = next(iter(rows.values()))
assert row["state"] == "uncertain"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(plan)
asyncio.run(run())
def test_periodic_lost_claim_response_recovers_exact_owner_as_uncertain(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
compare = engine.coordinator.store.compare_and_set
lose_background_reply = True
def commit_then_disconnect(kind, key, expected, value):
nonlocal lose_background_reply
result = compare(kind, key, expected, value)
document = json.loads(value)
if (result and lose_background_reply
and any(row.get("kind") == "background"
for row in document.get("claims", {}).values())):
lose_background_reply = False
raise RuntimeError("committed background claim reply lost")
return result
async def forbidden(*_args, **_kwargs):
pytest.fail("unknown background admission dispatched host work")
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", commit_then_disconnect)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_unavailable"
before = engine.coordinator.snapshot()
assert len(before) == 1
row = next(iter(before.values()))
assert row["state"] == "uncertain"
assert row["kind"] == "background"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(plan)
with pytest.raises(CoordinationAdmissionRejected) as blocked:
await vpc.reconcile("vpc-hk-1")
assert blocked.value.error_code == "coordination_uncertain"
assert engine.coordinator.snapshot() == before
asyncio.run(run())
def test_periodic_runtime_after_claim_commit_recovers_exact_owner_as_uncertain(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
claim = engine.coordinator.claim
def commit_then_raise(*args, **kwargs):
result = claim(*args, **kwargs)
if kwargs.get("kind") == "background":
raise RuntimeError("committed background claim reply lost")
return result
async def forbidden(*_args, **_kwargs):
pytest.fail("unknown background admission dispatched host work")
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
with pytest.raises(RuntimeError, match="claim reply lost"):
await vpc.reconcile("vpc-hk-1")
rows = engine.coordinator.snapshot()
assert len(rows) == 1
row = next(iter(rows.values()))
assert row["state"] == "uncertain"
assert row["kind"] == "background"
assert row["operation_id"] == OPERATION_ID
assert row["resources"] == []
assert row["metadata"] == identity(plan)
assert row["maintenance"] is False
asyncio.run(run())
def test_periodic_recovered_claim_retain_retries_precommit_failure(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
claim = engine.coordinator.claim
retain = engine.coordinator.retain
retain_calls = 0
def commit_then_raise(*args, **kwargs):
result = claim(*args, **kwargs)
if kwargs.get("kind") == "background":
raise RuntimeError("committed background claim reply lost")
return result
def fail_first_retain(value, *args, **kwargs):
nonlocal retain_calls
kind = (value.get("kind") if isinstance(value, dict)
else value.kind)
if kind == "background":
retain_calls += 1
if retain_calls == 1:
raise CoordinationError("coordination_unavailable", 503)
return retain(value, *args, **kwargs)
async def forbidden(*_args, **_kwargs):
pytest.fail("unknown background admission dispatched host work")
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
monkeypatch.setattr(engine.coordinator, "retain", fail_first_retain)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
with pytest.raises(RuntimeError, match="claim reply lost"):
await vpc.reconcile("vpc-hk-1")
assert retain_calls == 2
rows = engine.coordinator.snapshot()
assert len(rows) == 1
row = next(iter(rows.values()))
assert row["state"] == "uncertain"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(plan)
asyncio.run(run())
def test_periodic_cancel_during_final_snapshot_retains_root_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
body_completed = threading.Event()
snapshot_started = threading.Event()
release_snapshot = threading.Event()
snapshot = engine.coordinator.snapshot
async def host(*_args, **_kwargs):
body_completed.set()
def block_final_snapshot():
rows = snapshot()
if (body_completed.is_set() and len(rows) == 1
and next(iter(rows.values())).get("kind") == "background"):
snapshot_started.set()
assert release_snapshot.wait(5), "test did not release final snapshot"
return rows
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(engine.coordinator, "snapshot", block_final_snapshot)
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
try:
assert await asyncio.to_thread(snapshot_started.wait, 5)
task.cancel()
finally:
release_snapshot.set()
with pytest.raises(asyncio.CancelledError):
await task
rows = snapshot()
assert len(rows) == 1
row = next(iter(rows.values()))
assert row["state"] == "uncertain"
assert row["kind"] == "background"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(plan)
asyncio.run(run())
def test_periodic_cancel_after_committed_root_release_keeps_confirmed_settlement(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, _target = await seed(engine, store)
release = engine.coordinator.release
committed = threading.Event()
resume = threading.Event()
async def host(*_args, **_kwargs):
return None
def commit_then_block(target):
result = release(target)
kind = (target.get("kind") if isinstance(target, dict)
else target.kind)
if kind == "background" and result:
committed.set()
assert resume.wait(5), "test did not release committed root"
return result
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(engine.coordinator, "release", commit_then_block)
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
try:
assert await asyncio.to_thread(committed.wait, 5)
task.cancel()
finally:
resume.set()
with pytest.raises(asyncio.CancelledError):
await task
# The exact release CAS committed before cancellation. Recreating
# the deleted owner here would turn a confirmed settlement into a
# false uncertainty fence.
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
def test_periodic_failed_final_release_falls_back_to_uncertain_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
compare = engine.coordinator.store.compare_and_set
fail_release = True
def disconnect_before_release(kind, key, expected, value):
nonlocal fail_release
before = json.loads(expected)
after = json.loads(value)
background = [row for row in before.get("claims", {}).values()
if row.get("kind") == "background"]
if fail_release and len(background) == 1 and not after.get("claims"):
fail_release = False
raise RuntimeError("background release did not commit")
return compare(kind, key, expected, value)
async def host(*_args, **_kwargs):
return None
monkeypatch.setattr(
engine.coordinator.store, "compare_and_set", disconnect_before_release)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_unavailable"
rows = engine.coordinator.snapshot()
assert len(rows) == 1
row = next(iter(rows.values()))
assert row["state"] == "uncertain"
assert row["kind"] == "background"
assert row["operation_id"] == OPERATION_ID
assert row["metadata"] == identity(plan)
asyncio.run(run())
def test_periodic_active_child_leak_retains_root_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:leaked-child",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert rows[leaked.token]["state"] == "uncertain"
roots = [row for token, row in rows.items() if token != leaked.token]
assert len(roots) == 1
assert roots[0]["state"] == "uncertain"
assert roots[0]["kind"] == "background"
assert roots[0]["operation_id"] == OPERATION_ID
assert roots[0]["metadata"] == identity(plan)
asyncio.run(run())
def test_periodic_deterministic_failure_with_child_is_upgraded_to_uncertain(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:failed-child",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
raise VpcError("deterministic failure", 409,
"vpc_revision_conflict")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert all(row["state"] == "uncertain" for row in rows.values())
assert rows[leaked.token]["kind"] == "resource"
asyncio.run(run())
def test_periodic_child_retain_failure_retries_after_root_fence(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
child_retain_calls = 0
retain = engine.coordinator.retain
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:retry-child-retain",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
def fail_first_child_retain(target, *args, **kwargs):
nonlocal child_retain_calls
token = (target.token if hasattr(target, "token")
else target.get("token"))
if leaked is not None and token == leaked.token:
child_retain_calls += 1
if child_retain_calls == 1:
raise CoordinationError(
"coordination_unavailable", 503)
return retain(target, *args, **kwargs)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(
engine.coordinator, "retain", fail_first_child_retain)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_uncertain"
assert child_retain_calls == 2
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert all(row["state"] == "uncertain" for row in rows.values())
asyncio.run(run())
def test_periodic_cancel_during_child_retain_fences_all_children_and_root(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = []
blocked = False
entered = threading.Event()
resume = threading.Event()
retain = engine.coordinator.retain
async def host(*_args, **_kwargs):
context = current_execution()
for suffix in ("one", "two"):
leaked.append(engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=(f"test:cancel-child-retain-{suffix}",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
))
def commit_then_block(target, *args, **kwargs):
nonlocal blocked
token = (target.token if hasattr(target, "token")
else target.get("token"))
result = retain(target, *args, **kwargs)
if leaked and token == leaked[0].token and not blocked:
blocked = True
entered.set()
assert resume.wait(5), "test did not release child retain"
return result
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(engine.coordinator, "retain", commit_then_block)
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
try:
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
finally:
resume.set()
with pytest.raises(asyncio.CancelledError):
await task
rows = engine.coordinator.snapshot()
assert len(rows) == 3
assert all(rows[item.token]["state"] == "uncertain"
for item in leaked)
assert all(row["state"] == "uncertain" for row in rows.values())
asyncio.run(run())
def test_periodic_unknown_execution_never_returns_success(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
async def host(*_args, **_kwargs):
mark_execution_unknown()
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
with pytest.raises(CoordinationError) as raised:
await vpc.reconcile("vpc-hk-1")
assert raised.value.error_code == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert_retained_claim_family(
rows, plan, root_kind="background",
member_uuids=(INSTANCE_A, INSTANCE_B))
asyncio.run(run())
def test_http_active_child_leak_retains_root_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:http-leaked-child",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert rows[leaked.token]["state"] == "uncertain"
roots = [row for token, row in rows.items() if token != leaked.token]
assert len(roots) == 1
assert roots[0]["state"] == "uncertain"
assert roots[0]["kind"] == "request"
assert roots[0]["operation_id"] == OPERATION_ID
assert roots[0]["metadata"] == identity(plan)
asyncio.run(run())
def test_http_child_retain_failure_still_retains_root_owner(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
child_retain_calls = 0
retain = engine.coordinator.retain
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:http-child-retain-failure",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
def fail_child_retain(claim, *args, **kwargs):
nonlocal child_retain_calls
token = (claim.token if hasattr(claim, "token") else
claim.get("token"))
if leaked is not None and token == leaked.token:
child_retain_calls += 1
if child_retain_calls == 1:
raise CoordinationError(
"coordination_unavailable", 503)
return retain(claim, *args, **kwargs)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(engine.coordinator, "retain", fail_child_retain)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert child_retain_calls == 2
assert rows[leaked.token]["state"] == "uncertain"
roots = [row for token, row in rows.items() if token != leaked.token]
assert len(roots) == 1
assert roots[0]["state"] == "uncertain"
assert roots[0]["operation_id"] == OPERATION_ID
assert roots[0]["metadata"] == identity(plan)
asyncio.run(run())
def test_http_unknown_execution_without_residual_returns_503(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
async def host(*_args, **_kwargs):
mark_execution_unknown()
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert_retained_claim_family(
rows, plan, root_kind="request",
member_uuids=(INSTANCE_A, INSTANCE_B))
asyncio.run(run())
def test_http_lost_child_release_reply_with_absent_token_returns_503(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
release = engine.coordinator.release
lost_reply = False
async def host(*_args, **_kwargs):
return None
def commit_child_release_then_disconnect(claim):
nonlocal lost_reply
released = release(claim)
if claim.kind == "resource" and not lost_reply:
lost_reply = True
mark_execution_unknown()
raise CoordinationError(
"coordination_unavailable", 503)
return released
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(
engine.coordinator, "release",
commit_child_release_then_disconnect)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert lost_reply is True
rows = engine.coordinator.snapshot()
assert_retained_claim_family(
rows, plan, root_kind="request",
member_uuids=(INSTANCE_A,))
assert all(
f"vm:host-a:{INSTANCE_B}" not in row["resources"]
for row in rows.values())
asyncio.run(run())
def test_http_exception_retains_residual_child_and_root(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:http-exception-child",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
raise VpcError("deterministic failure", 409,
"vpc_revision_conflict")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
rows = engine.coordinator.snapshot()
assert len(rows) == 2
assert rows[leaked.token]["state"] == "uncertain"
roots = [row for token, row in rows.items()
if token != leaked.token]
assert len(roots) == 1
assert roots[0]["state"] == "uncertain"
asyncio.run(run())
def test_http_unhandled_exception_retains_residual_child_and_root(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, target = await seed(engine, store)
plan = parse_vpc_plan(target)
leaked = None
async def host(*_args, **_kwargs):
nonlocal leaked
context = current_execution()
leaked = engine.coordinator.claim(
operation_id=OPERATION_ID,
resources=("test:http-unhandled-exception-child",),
execution_id=context.claim.execution_id,
kind="resource",
metadata=identity(plan),
)
raise RuntimeError("unhandled handler failure")
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
rows = engine.coordinator.snapshot()
# The exception escapes before the nested catalog/member locks can
# finish, so every claim from this execution must be fenced.
assert len(rows) == 5
assert rows[leaked.token]["state"] == "uncertain"
assert all(row["state"] == "uncertain"
and row["operation_id"] == OPERATION_ID
and row["metadata"] == identity(plan)
for row in rows.values())
roots = [row for row in rows.values()
if row["kind"] == "request"]
assert len(roots) == 1
asyncio.run(run())
def test_http_false_root_release_is_read_back_and_returns_503(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, _target = await seed(engine, store)
release = engine.coordinator.release
failed = False
async def host(*_args, **_kwargs):
return None
def false_root_release(claim):
nonlocal failed
kind = (claim.kind if hasattr(claim, "kind") else
claim.get("kind"))
if kind == "request" and not failed:
failed = True
return False
return release(claim)
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(
engine.coordinator, "release", false_root_release)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert failed is True
rows = engine.coordinator.snapshot()
assert len(rows) == 1
root = next(iter(rows.values()))
assert root["state"] == "uncertain"
assert root["operation_id"] == OPERATION_ID
asyncio.run(run())
def test_http_lost_root_release_reply_reads_absence_and_returns_503(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
vpc, _target = await seed(engine, store)
release = engine.coordinator.release
lost_reply = False
async def host(*_args, **_kwargs):
return None
def commit_root_release_then_disconnect(claim):
nonlocal lost_reply
released = release(claim)
kind = (claim.kind if hasattr(claim, "kind") else
claim.get("kind"))
if kind == "request" and not lost_reply:
lost_reply = True
raise CoordinationError(
"coordination_unavailable", 503)
return released
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
monkeypatch.setattr(
engine.coordinator, "release",
commit_root_release_then_disconnect)
status, result = await request(
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
assert status == 503, result
assert result["error_code"] == "coordination_uncertain"
assert lost_reply is True
assert engine.coordinator.snapshot() == {}
asyncio.run(run())