You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1752 lines
74 KiB
Python
1752 lines
74 KiB
Python
"""VPC claims bind exact immutable target identity before host admission."""
|
|
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.coordination import (
|
|
CoordinationAdmissionRejected, CoordinationClaimRejected, CoordinationError,
|
|
DurableCoordinator, current_execution, execution_context,
|
|
mark_execution_unknown,
|
|
)
|
|
from virsh_panel.vpc import (
|
|
VpcCoordinator, VpcError, parse_vpc_delete_plan, parse_vpc_plan,
|
|
)
|
|
from virsh_panel.vpc_dataplane import derive_private_mac
|
|
from test_durable_fabric_jobs import FabricStore
|
|
from test_engine_boundary import SUPER, cloud_engine_request
|
|
from test_engine_coordination import local_backends, request, running_engine
|
|
from test_vpc import (
|
|
FakeFabric, INSTANCE_A, INSTANCE_B, confirm_delete_hosts, delete_payload,
|
|
payload,
|
|
)
|
|
from test_vpc_admission_coordination import vpc_payload
|
|
|
|
|
|
OPERATION_ID = "a" * 32
|
|
|
|
|
|
def identity(plan):
|
|
return {"job_kind": "vpc_operation", "vpc_id": plan.vpc_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest}
|
|
|
|
|
|
def assert_retained_claim_family(rows, plan, *, root_kind, member_uuids):
|
|
expected_kinds = {
|
|
(): root_kind,
|
|
("catalog:network",): "resource",
|
|
**{
|
|
(f"vm:host-a:{instance_uuid}",): "resource"
|
|
for instance_uuid in member_uuids
|
|
},
|
|
}
|
|
by_resources = {
|
|
tuple(row["resources"]): row
|
|
for row in rows.values()
|
|
}
|
|
assert len(rows) == len(expected_kinds)
|
|
assert set(by_resources) == set(expected_kinds)
|
|
for resources, expected_kind in expected_kinds.items():
|
|
row = by_resources[resources]
|
|
assert row["kind"] == expected_kind
|
|
assert row["state"] == "uncertain"
|
|
assert row["operation_id"] == plan.operation_id
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
|
|
async def explicit_request(engine, method, target, value=None):
|
|
actor = {**SUPER, "vpc_reconcile_explicit": True}
|
|
body = b"" if value is None else json.dumps(value).encode()
|
|
return await cloud_engine_request(
|
|
engine, method, target, body, actor,
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
|
|
|
|
def retained_root(coordinator, plan, *, kind="background"):
|
|
claim = coordinator.claim(
|
|
operation_id=plan.operation_id, kind=kind, metadata=identity(plan))
|
|
assert coordinator.retain(claim)
|
|
return coordinator.snapshot()[claim.token]
|
|
|
|
|
|
def coordination_document(store):
|
|
return json.loads(store.get("coordination", "admission-v1"))
|
|
|
|
|
|
def retiring_members(value):
|
|
plan = parse_vpc_plan(value)
|
|
return tuple({
|
|
"member_id": member.member_id,
|
|
"instance_uuid": member.instance_uuid,
|
|
"host_id": member.host_id,
|
|
"private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": (member.private_mac
|
|
or derive_private_mac(plan.vpc_id, member.member_id)),
|
|
} for member in plan.members)
|
|
|
|
|
|
async def seed(engine, store, *, empty=False):
|
|
target = payload(hosts=[], members=[], operation_id=OPERATION_ID) if empty else {
|
|
**vpc_payload(), "operation_id": OPERATION_ID}
|
|
vpc = VpcCoordinator(store, FakeFabric(hosts=("host-a",)),
|
|
catalog_lock=engine._network_catalog_mutation_lock,
|
|
vm_lock=engine._target_lock, coordinator=engine.coordinator)
|
|
await vpc.apply(target)
|
|
assert engine.coordinator.snapshot() == {}
|
|
engine.vpc = vpc
|
|
return vpc, target
|
|
|
|
|
|
@pytest.mark.parametrize("old_kind", ["background", "request"])
|
|
def test_explicit_http_apply_atomically_takes_over_exact_retained_root(
|
|
tmp_path, monkeypatch, old_kind):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
old = retained_root(engine.coordinator, plan, kind=old_kind)
|
|
seen = []
|
|
|
|
async def host(*_args, **_kwargs):
|
|
rows = list(engine.coordinator.snapshot().values())
|
|
seen.extend(rows)
|
|
assert rows
|
|
assert all(row["operation_id"] == plan.operation_id
|
|
and row["metadata"] == identity(plan)
|
|
and "explicit_vpc_reconcile" not in row["metadata"]
|
|
for row in rows)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 200, result
|
|
assert seen
|
|
assert engine.coordinator.snapshot() == {}
|
|
document = coordination_document(store)
|
|
receipts = [item for item in document["resolutions"]
|
|
if item.get("kind") == "resolve_and_bind_v1"]
|
|
assert len(receipts) == 1
|
|
receipt = receipts[0]
|
|
assert receipt["token"] == old["token"]
|
|
assert receipt["operation_id"] == plan.operation_id
|
|
assert receipt["replacement_token"] != old["token"]
|
|
assert set(receipt) == {
|
|
"kind", "token", "operation_id", "resolved_by",
|
|
"resolved_at", "evidence", "source_claim_sha256",
|
|
"replacement_token", "replacement_identity_sha256",
|
|
}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_explicit_http_apply_without_retained_root_consumes_marker(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
seen = []
|
|
|
|
async def host(*_args, **_kwargs):
|
|
seen.extend(engine.coordinator.snapshot().values())
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 200, result
|
|
assert seen
|
|
assert all(row["metadata"] == identity(plan)
|
|
and "explicit_vpc_reconcile" not in row["metadata"]
|
|
for row in seen)
|
|
assert engine.coordinator.snapshot() == {}
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_apply_without_explicit_marker_keeps_exact_retained_root(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
old = retained_root(
|
|
engine.coordinator, parse_vpc_plan(target))
|
|
before = engine.coordinator.snapshot()
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unmarked retry reached host dispatch")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert engine.coordinator.snapshot() == before
|
|
assert old["token"] in before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_explicit_http_delete_takes_over_only_durable_delete_target(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, applied = await seed(engine, store)
|
|
target = delete_payload(
|
|
operation_id="b" * 32,
|
|
retiring_members=retiring_members(applied),
|
|
)
|
|
plan = parse_vpc_delete_plan(target)[0]
|
|
release = engine.coordinator.release
|
|
failed_root_release = False
|
|
|
|
async def confirmed(plan_value, record, *_args, **_kwargs):
|
|
confirm_delete_hosts(plan_value, record)
|
|
|
|
def lose_background_release(claim):
|
|
nonlocal failed_root_release
|
|
kind = (claim.get("kind") if isinstance(claim, dict)
|
|
else claim.kind)
|
|
if kind == "background" and not failed_root_release:
|
|
failed_root_release = True
|
|
raise RuntimeError("lost background release response")
|
|
return release(claim)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane", confirmed)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "release", lose_background_release)
|
|
with pytest.raises(RuntimeError):
|
|
await vpc.delete("vpc-hk-1", target)
|
|
assert failed_root_release
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
old = next(iter(rows.values()))
|
|
assert old["state"] == "uncertain"
|
|
assert old["operation_id"] == plan.operation_id
|
|
assert old["metadata"] == identity(plan)
|
|
assert store.get("vpc_desired", "vpc-hk-1") is None
|
|
|
|
monkeypatch.setattr(engine.coordinator, "release", release)
|
|
monkeypatch.setattr(
|
|
vpc, "_reconcile_dataplane",
|
|
lambda *_args, **_kwargs: pytest.fail(
|
|
"confirmed delete receipt retried host dispatch"))
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
|
|
assert status == 200, result
|
|
assert result["deleted"] is True
|
|
assert engine.coordinator.snapshot() == {}
|
|
receipts = coordination_document(store)["resolutions"]
|
|
assert any(item.get("kind") == "resolve_and_bind_v1"
|
|
and item.get("token") == old["token"]
|
|
for item in receipts)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_delete_without_explicit_marker_keeps_retained_delete_root(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, applied = await seed(engine, store)
|
|
target = delete_payload(
|
|
operation_id="c" * 32,
|
|
retiring_members=retiring_members(applied),
|
|
)
|
|
plan = parse_vpc_delete_plan(target)[0]
|
|
release = engine.coordinator.release
|
|
failed_root_release = False
|
|
|
|
async def confirmed(plan_value, record, *_args, **_kwargs):
|
|
confirm_delete_hosts(plan_value, record)
|
|
|
|
def lose_background_release(claim):
|
|
nonlocal failed_root_release
|
|
kind = (claim.get("kind") if isinstance(claim, dict)
|
|
else claim.kind)
|
|
if kind == "background" and not failed_root_release:
|
|
failed_root_release = True
|
|
raise RuntimeError("lost background release response")
|
|
return release(claim)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane", confirmed)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "release", lose_background_release)
|
|
with pytest.raises(RuntimeError):
|
|
await vpc.delete("vpc-hk-1", target)
|
|
assert failed_root_release
|
|
monkeypatch.setattr(engine.coordinator, "release", release)
|
|
before = engine.coordinator.snapshot()
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unmarked delete retry reached host dispatch")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane", forbidden)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert engine.coordinator.snapshot() == before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_explicit_delete_does_not_take_over_root_with_residual_child(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, applied = await seed(engine, store)
|
|
target = delete_payload(
|
|
operation_id="e" * 32,
|
|
retiring_members=retiring_members(applied),
|
|
)
|
|
|
|
async def interrupted(*_args, **_kwargs):
|
|
raise CoordinationError("coordination_unavailable", 503)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane", interrupted)
|
|
with pytest.raises(CoordinationError):
|
|
await vpc.delete("vpc-hk-1", target)
|
|
before = engine.coordinator.snapshot()
|
|
assert len(before) == 2
|
|
execution_ids = {row["execution_id"] for row in before.values()}
|
|
assert len(execution_ids) == 1
|
|
assert any(row["kind"] == "background" for row in before.values())
|
|
assert any(row["kind"] == "resource" for row in before.values())
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("residual-child delete reached host dispatch")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane", forbidden)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert engine.coordinator.snapshot() == before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_explicit_apply_revalidates_durable_target_before_handoff(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, stale = await seed(engine, store)
|
|
stale_plan = parse_vpc_plan(stale)
|
|
old = retained_root(engine.coordinator, stale_plan)
|
|
newer = {
|
|
**stale,
|
|
"desired_revision": stale_plan.desired_revision + 1,
|
|
"operation_id": "f" * 32,
|
|
}
|
|
await vpc.apply(newer)
|
|
before = engine.coordinator.snapshot()
|
|
assert set(before) == {old["token"]}
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("stale retained target reached host dispatch")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", stale)
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert engine.coordinator.snapshot() == before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["before", "after", "false_after"])
|
|
def test_explicit_handoff_cas_fault_is_read_back_fenced_and_retryable(
|
|
tmp_path, monkeypatch, fault):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
old = retained_root(engine.coordinator, plan)
|
|
compare = engine.coordinator.store.compare_and_set
|
|
triggered = False
|
|
host_calls = 0
|
|
|
|
def fail_handoff(kind, key, expected, value):
|
|
nonlocal triggered
|
|
desired = json.loads(value)
|
|
handoff = any(
|
|
item.get("kind") == "resolve_and_bind_v1"
|
|
and item.get("token") == old["token"]
|
|
for item in desired.get("resolutions", []))
|
|
if handoff and not triggered:
|
|
triggered = True
|
|
if fault == "before":
|
|
raise RuntimeError("handoff did not commit")
|
|
assert compare(kind, key, expected, value)
|
|
if fault == "after":
|
|
raise RuntimeError("handoff response lost")
|
|
return False
|
|
return compare(kind, key, expected, value)
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal host_calls
|
|
host_calls += 1
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", fail_handoff)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 503, result
|
|
assert result["error_code"] in {
|
|
"coordination_uncertain", "coordination_unavailable"}
|
|
assert triggered and host_calls == 0
|
|
rows = engine.coordinator.snapshot()
|
|
assert old["token"] not in rows
|
|
assert len(rows) == 1
|
|
root = next(iter(rows.values()))
|
|
assert root["state"] == "uncertain"
|
|
assert root["kind"] == "request"
|
|
assert root["resources"] == []
|
|
assert root["operation_id"] == plan.operation_id
|
|
assert root["metadata"] == identity(plan)
|
|
receipts = [item for item in coordination_document(store)["resolutions"]
|
|
if item.get("kind") == "resolve_and_bind_v1"]
|
|
assert len(receipts) == 1
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", compare)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 200, result
|
|
assert host_calls == 1
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["before", "after", "false_after"])
|
|
def test_marker_only_bind_cas_fault_strips_capability_and_is_retryable(
|
|
tmp_path, monkeypatch, fault):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, _store, _services):
|
|
vpc, target = await seed(engine, _store)
|
|
plan = parse_vpc_plan(target)
|
|
compare = engine.coordinator.store.compare_and_set
|
|
triggered = False
|
|
host_calls = 0
|
|
|
|
def fail_bind(kind, key, expected, value):
|
|
nonlocal triggered
|
|
before = json.loads(expected).get("claims", {})
|
|
after = json.loads(value).get("claims", {})
|
|
bound = any(
|
|
token in before
|
|
and before[token].get("operation_id") is None
|
|
and before[token].get("metadata")
|
|
== {"explicit_vpc_reconcile": True}
|
|
and row.get("operation_id") == plan.operation_id
|
|
and row.get("metadata") == identity(plan)
|
|
for token, row in after.items())
|
|
if bound and not triggered:
|
|
triggered = True
|
|
if fault == "before":
|
|
raise RuntimeError("bind did not commit")
|
|
assert compare(kind, key, expected, value)
|
|
if fault == "after":
|
|
raise RuntimeError("bind response lost")
|
|
return False
|
|
return compare(kind, key, expected, value)
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal host_calls
|
|
host_calls += 1
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", fail_bind)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 503, result
|
|
assert triggered and host_calls == 0
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
root = next(iter(rows.values()))
|
|
assert root["state"] == "uncertain"
|
|
assert root["operation_id"] == plan.operation_id
|
|
assert root["metadata"] == identity(plan)
|
|
assert "explicit_vpc_reconcile" not in root["metadata"]
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", compare)
|
|
status, result = await explicit_request(
|
|
engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 200, result
|
|
assert host_calls == 1
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("phase", ["handoff", "bind"])
|
|
def test_explicit_admission_cancellation_fences_exact_bound_root(
|
|
tmp_path, monkeypatch, phase):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
old = (retained_root(engine.coordinator, plan)
|
|
if phase == "handoff" else None)
|
|
method_name = ("resolve_and_bind" if phase == "handoff"
|
|
else "bind_explicit_operation")
|
|
original = getattr(engine.coordinator, method_name)
|
|
entered = threading.Event()
|
|
resume = threading.Event()
|
|
|
|
def commit_then_block(*args, **kwargs):
|
|
result = original(*args, **kwargs)
|
|
entered.set()
|
|
assert resume.wait(5), "test did not release admission CAS"
|
|
return result
|
|
|
|
monkeypatch.setattr(engine.coordinator, method_name, commit_then_block)
|
|
parent = engine.coordinator.claim(
|
|
kind="request", metadata={"explicit_vpc_reconcile": True})
|
|
|
|
async def apply_in_parent():
|
|
with execution_context(engine.coordinator, parent):
|
|
return await vpc.apply(target)
|
|
|
|
task = asyncio.create_task(apply_in_parent())
|
|
try:
|
|
assert await asyncio.to_thread(entered.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
resume.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
if old is not None:
|
|
assert old["token"] not in rows
|
|
assert len(rows) == 1
|
|
root = next(iter(rows.values()))
|
|
assert root["token"] == parent.token
|
|
assert root["state"] == "uncertain"
|
|
assert root["operation_id"] == plan.operation_id
|
|
assert root["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("child_execution", ["old", "replacement"])
|
|
def test_atomic_handoff_rejects_any_child_of_either_execution(
|
|
tmp_path, child_execution):
|
|
store = FabricStore()
|
|
coordinator = DurableCoordinator(store, owner_id="d" * 64)
|
|
target = payload(operation_id=OPERATION_ID)
|
|
plan = parse_vpc_plan(target)
|
|
old_claim = coordinator.claim(
|
|
operation_id=plan.operation_id, kind="background",
|
|
metadata=identity(plan))
|
|
parent = coordinator.claim(
|
|
kind="request", metadata={"explicit_vpc_reconcile": True})
|
|
execution_id = (old_claim.execution_id if child_execution == "old"
|
|
else parent.execution_id)
|
|
child = coordinator.claim(
|
|
operation_id=(plan.operation_id if child_execution == "old" else None),
|
|
resources=(f"test:{child_execution}-execution-child",),
|
|
execution_id=execution_id, kind="resource",
|
|
metadata=(identity(plan) if child_execution == "old"
|
|
else {"explicit_vpc_reconcile": True}))
|
|
assert coordinator.retain(old_claim)
|
|
before = coordinator.snapshot()
|
|
old = before[old_claim.token]
|
|
|
|
with pytest.raises(CoordinationClaimRejected):
|
|
coordinator.resolve_and_bind(
|
|
old, parent, "exact retained root evidence",
|
|
operation_id=plan.operation_id, metadata=identity(plan))
|
|
|
|
assert coordinator.snapshot() == before
|
|
assert child.token in before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("invalid", ["kind", "resources", "metadata"])
|
|
def test_atomic_handoff_rejects_noncanonical_retained_root(invalid):
|
|
store = FabricStore()
|
|
coordinator = DurableCoordinator(store, owner_id="d" * 64)
|
|
plan = parse_vpc_plan(payload(operation_id=OPERATION_ID))
|
|
kwargs = {
|
|
"operation_id": plan.operation_id,
|
|
"kind": "background",
|
|
"metadata": identity(plan),
|
|
}
|
|
if invalid == "kind":
|
|
kwargs["kind"] = "resource"
|
|
elif invalid == "resources":
|
|
kwargs["resources"] = ("test:noncanonical-root",)
|
|
else:
|
|
kwargs["metadata"] = {**identity(plan), "desired_revision": 999}
|
|
old_claim = coordinator.claim(**kwargs)
|
|
assert coordinator.retain(old_claim)
|
|
parent = coordinator.claim(
|
|
kind="request", metadata={"explicit_vpc_reconcile": True})
|
|
before = coordinator.snapshot()
|
|
|
|
with pytest.raises(CoordinationClaimRejected):
|
|
coordinator.resolve_and_bind(
|
|
before[old_claim.token], parent,
|
|
"exact retained root evidence",
|
|
operation_id=plan.operation_id, metadata=identity(plan))
|
|
|
|
assert coordinator.snapshot() == before
|
|
assert coordination_document(store)["resolutions"] == []
|
|
|
|
|
|
def test_handoff_readback_rejects_mismatched_related_receipt():
|
|
store = FabricStore()
|
|
coordinator = DurableCoordinator(store, owner_id="d" * 64)
|
|
plan = parse_vpc_plan(payload(operation_id=OPERATION_ID))
|
|
old_claim = coordinator.claim(
|
|
operation_id=plan.operation_id, kind="background",
|
|
metadata=identity(plan))
|
|
assert coordinator.retain(old_claim)
|
|
old = coordinator.snapshot()[old_claim.token]
|
|
parent = coordinator.claim(
|
|
kind="request", metadata={"explicit_vpc_reconcile": True})
|
|
evidence = "exact retained root evidence"
|
|
coordinator.resolve_and_bind(
|
|
old, parent, evidence,
|
|
operation_id=plan.operation_id, metadata=identity(plan))
|
|
|
|
document = coordination_document(store)
|
|
document["resolutions"][-1]["evidence"] = "different evidence"
|
|
store.set("coordination", "admission-v1", json.dumps(document))
|
|
outcome, current = coordinator.classify_resolve_and_bind(
|
|
old, parent, evidence,
|
|
operation_id=plan.operation_id, metadata=identity(plan))
|
|
assert outcome == "mixed"
|
|
assert current["token"] == parent.token
|
|
|
|
|
|
@pytest.mark.parametrize("entry", ["apply", "reconcile"])
|
|
@pytest.mark.parametrize("unknown", [False, True])
|
|
def test_http_vpc_claims_use_signed_target_and_never_generic_vm_receipt(
|
|
tmp_path, monkeypatch, entry, unknown):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
seen = []
|
|
# An unrelated generic VM receipt must not settle VPC work just
|
|
# because it happens to share the opaque operation identifier.
|
|
store.set("operation", OPERATION_ID, json.dumps({"mutation_started": False}))
|
|
|
|
async def host(*_args, **_kwargs):
|
|
rows = list(engine.coordinator.snapshot().values())
|
|
seen.append(rows)
|
|
assert len(rows) == 4 # request, catalog, both member VM locks
|
|
assert {row["operation_id"] for row in rows} == {OPERATION_ID}
|
|
assert all(row["metadata"] == identity(plan) for row in rows)
|
|
assert current_execution().claim.operation_id == OPERATION_ID
|
|
if unknown:
|
|
raise CoordinationError("coordination_uncertain", 503)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await request(engine, "POST",
|
|
"/internal/v1/vpcs/apply" if entry == "apply" else "/internal/v1/vpcs/vpc-hk-1/reconcile",
|
|
target if entry == "apply" else {}, operation_id="actor-id-is-not-the-vpc-target")
|
|
assert seen
|
|
assert status == (503 if unknown else 200), result
|
|
rows = list(engine.coordinator.snapshot().values())
|
|
if unknown:
|
|
assert len(rows) == 4
|
|
assert all(row["state"] == "uncertain" and row["operation_id"] == OPERATION_ID
|
|
and row["metadata"] == identity(plan) for row in rows)
|
|
else:
|
|
assert rows == []
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("unknown", [False, True])
|
|
def test_periodic_vpc_claims_are_named_and_unknown_retry_keeps_original_fence(
|
|
tmp_path, monkeypatch, unknown):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
seen = []
|
|
|
|
async def host(*_args, **_kwargs):
|
|
rows = list(engine.coordinator.snapshot().values())
|
|
seen.append(rows)
|
|
assert len(rows) == 4
|
|
assert any(row["kind"] == "background" for row in rows)
|
|
assert all(row["operation_id"] == OPERATION_ID
|
|
and row["metadata"] == identity(plan) for row in rows)
|
|
if unknown:
|
|
raise CoordinationError("coordination_uncertain", 503)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
if unknown:
|
|
with pytest.raises(CoordinationError):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
before = engine.coordinator.snapshot()
|
|
assert len(before) == 4
|
|
assert all(row["state"] == "uncertain" for row in before.values())
|
|
with pytest.raises(CoordinationAdmissionRejected):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
await vpc.reconcile_all()
|
|
assert engine.coordinator.snapshot() == before
|
|
assert len(seen) == 1
|
|
else:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert seen and engine.coordinator.snapshot() == {}
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["active", "uncertain"])
|
|
@pytest.mark.parametrize("entry", ["http", "periodic"])
|
|
def test_same_operation_admission_rejection_does_not_create_anonymous_uncertainty(
|
|
tmp_path, monkeypatch, state, entry):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
owner = DurableCoordinator(store)
|
|
blocker = owner.claim(operation_id=OPERATION_ID, kind="resource")
|
|
if state == "uncertain": owner.retain(blocker)
|
|
before = owner.snapshot()
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("rejected operation reached host dispatch")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
if entry == "http":
|
|
status, result = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == (503 if state == "uncertain" else 409), result
|
|
else:
|
|
with pytest.raises(CoordinationAdmissionRejected):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert owner.snapshot() == before
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_delete_claim_binds_the_delete_target_before_receipt_write(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, _ = await seed(engine, store, empty=True)
|
|
target = delete_payload(hosts=(), operation_id="b" * 32)
|
|
observed = []
|
|
original = vpc._save
|
|
|
|
def save(kind, key, value):
|
|
if kind == "vpc_deleted":
|
|
rows = list(engine.coordinator.snapshot().values())
|
|
observed.extend(rows)
|
|
assert len(rows) == 2
|
|
assert all(row["operation_id"] == target["operation_id"] for row in rows)
|
|
assert all(row["metadata"]["vpc_id"] == "vpc-hk-1"
|
|
and row["metadata"]["desired_revision"] == 2 for row in rows)
|
|
return original(kind, key, value)
|
|
|
|
monkeypatch.setattr(vpc, "_save", save)
|
|
status, result = await request(engine, "POST", "/internal/v1/vpcs/vpc-hk-1/delete", target)
|
|
assert status == 200 and result["deleted"] is True, result
|
|
assert observed and engine.coordinator.snapshot() == {}
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("entry", ["http", "periodic"])
|
|
def test_legacy_anonymous_catalog_claim_is_preserved_without_new_uncertainty(
|
|
tmp_path, monkeypatch, entry):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
owner = DurableCoordinator(store)
|
|
old = owner.claim(resources=("catalog:network",), kind="resource")
|
|
owner.retain(old)
|
|
before = owner.snapshot()
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("legacy unknown owner was bypassed")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
if entry == "http":
|
|
status, result = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status == 503 and result["error_code"] == "coordination_uncertain", result
|
|
else:
|
|
with pytest.raises(CoordinationAdmissionRejected):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert owner.snapshot() == before
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["missing_id", "invalid_id", "digest", "revision"])
|
|
def test_invalid_signed_vpc_envelope_never_binds_or_dispatches(tmp_path, monkeypatch, fault):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
target = dict(target)
|
|
if fault == "missing_id": target.pop("operation_id")
|
|
elif fault == "invalid_id": target["operation_id"] = "../unsafe"
|
|
elif fault == "digest": target["desired_digest"] = "0" * 64
|
|
else: target["desired_revision"] = 0
|
|
bound = []
|
|
bind = engine.coordinator.bind_operation
|
|
|
|
def track(*args, **kwargs):
|
|
bound.append(True)
|
|
return bind(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(engine.coordinator, "bind_operation", track)
|
|
status, _ = await request(engine, "POST", "/internal/v1/vpcs/apply", target)
|
|
assert status in {400, 409, 422}
|
|
assert not bound and engine.coordinator.snapshot() == {}
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_reconcile_identity_change_before_catalog_admission_is_rejected(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
bind = engine.coordinator.bind_operation
|
|
|
|
def supersede(*args, **kwargs):
|
|
result = bind(*args, **kwargs)
|
|
record = json.loads(store.get("vpc_desired", "vpc-hk-1"))
|
|
record["operation_id"] = "b" * 32
|
|
store.set("vpc_desired", "vpc-hk-1", json.dumps(record))
|
|
return result
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("stale bound identity reached host dispatch")
|
|
|
|
monkeypatch.setattr(engine.coordinator, "bind_operation", supersede)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
status, result = await request(engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 409 and result["error_code"] == "vpc_revision_conflict", result
|
|
assert engine.coordinator.snapshot() == {}
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_bind_operation_is_immutable_and_keeps_ownership_and_extra_metadata():
|
|
owner = DurableCoordinator(FabricStore())
|
|
claim = owner.claim(kind="request", metadata={"audit": "preserved"})
|
|
metadata = identity(parse_vpc_plan(payload(operation_id=OPERATION_ID)))
|
|
bound = owner.bind_operation(claim, OPERATION_ID, metadata)
|
|
assert (bound.token, bound.execution_id, bound.owner_id) == (claim.token, claim.execution_id, claim.owner_id)
|
|
assert bound.metadata == {"audit": "preserved", **metadata}
|
|
assert owner.bind_operation(bound, OPERATION_ID, metadata) == bound
|
|
before = owner.snapshot()
|
|
for operation, changed in [("b" * 32, metadata), (OPERATION_ID, {**metadata, "vpc_id": "another-vpc"})]:
|
|
with pytest.raises(CoordinationClaimRejected):
|
|
owner.bind_operation(bound, operation, changed)
|
|
assert owner.snapshot() == before
|
|
assert owner.release(claim) and owner.snapshot() == {}
|
|
|
|
|
|
def test_lost_bind_response_preserves_committed_operation_identity():
|
|
class LostReply(FabricStore):
|
|
fail = False
|
|
def compare_and_set(self, kind, key, expected, value):
|
|
result = super().compare_and_set(kind, key, expected, value)
|
|
if result and self.fail:
|
|
self.fail = False
|
|
raise RuntimeError("committed bind reply lost")
|
|
return result
|
|
store = LostReply()
|
|
owner = DurableCoordinator(store)
|
|
claim = owner.claim(kind="request")
|
|
store.fail = True
|
|
with execution_context(owner, claim):
|
|
with pytest.raises(CoordinationError):
|
|
owner.bind_operation(claim, OPERATION_ID, {"job_kind": "vpc_operation"})
|
|
assert current_execution().execution_unknown.is_set()
|
|
assert owner.snapshot()[claim.token]["operation_id"] == OPERATION_ID
|
|
|
|
|
|
@pytest.mark.parametrize("entry", ["http", "periodic"])
|
|
def test_legacy_anonymous_empty_claim_is_not_borrowed_or_a_vpc_blocker(
|
|
tmp_path, entry):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, _target = await seed(engine, store)
|
|
legacy_owner = DurableCoordinator(store)
|
|
legacy = legacy_owner.claim(kind="request")
|
|
assert legacy_owner.retain(legacy)
|
|
before = legacy_owner.snapshot()
|
|
row = before[legacy.token]
|
|
assert row["state"] == "uncertain"
|
|
assert row["operation_id"] is None
|
|
assert row["resources"] == []
|
|
assert row["metadata"] == {}
|
|
|
|
if entry == "http":
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 200, result
|
|
else:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
|
|
# The legacy row has no target identity or resource fence, so it
|
|
# cannot safely be borrowed by this operation. It also does not
|
|
# intersect the exact VPC operation/catalog claims admitted above.
|
|
assert legacy_owner.snapshot() == before
|
|
# It still blocks process-wide maintenance until an operator can
|
|
# supply explicit evidence to DurableCoordinator.resolve().
|
|
with pytest.raises(CoordinationClaimRejected) as blocked:
|
|
legacy_owner.claim(maintenance=True, kind="update")
|
|
assert blocked.value.error_code == "coordination_uncertain"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_cancel_after_claim_commit_retains_exact_owner(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
committed = threading.Event()
|
|
reply = threading.Event()
|
|
claim = engine.coordinator.claim
|
|
|
|
def delayed_reply(*args, **kwargs):
|
|
result = claim(*args, **kwargs)
|
|
if kwargs.get("kind") == "background":
|
|
committed.set()
|
|
assert reply.wait(5), "test did not release committed claim response"
|
|
return result
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("cancelled admission dispatched host work")
|
|
|
|
monkeypatch.setattr(engine.coordinator, "claim", delayed_reply)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
|
|
try:
|
|
assert await asyncio.to_thread(committed.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
reply.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
before = engine.coordinator.snapshot()
|
|
assert len(before) == 1
|
|
row = next(iter(before.values()))
|
|
assert row["state"] == "uncertain" and row["kind"] == "background"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(parse_vpc_plan(target))
|
|
with pytest.raises(CoordinationAdmissionRejected):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert engine.coordinator.snapshot() == before
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_cancel_during_lost_claim_recovery_fences_exact_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
claim = engine.coordinator.claim
|
|
snapshot = engine.coordinator.snapshot
|
|
entered = threading.Event()
|
|
resume = threading.Event()
|
|
blocked = False
|
|
|
|
def commit_then_raise(*args, **kwargs):
|
|
result = claim(*args, **kwargs)
|
|
if kwargs.get("kind") == "background":
|
|
raise RuntimeError("committed background claim reply lost")
|
|
return result
|
|
|
|
def block_recovery_snapshot():
|
|
nonlocal blocked
|
|
rows = snapshot()
|
|
if (not blocked and len(rows) == 1
|
|
and next(iter(rows.values())).get("kind") == "background"):
|
|
blocked = True
|
|
entered.set()
|
|
assert resume.wait(5), "test did not release recovery snapshot"
|
|
return rows
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unknown background admission dispatched host work")
|
|
|
|
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
|
|
monkeypatch.setattr(engine.coordinator, "snapshot", block_recovery_snapshot)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
|
|
try:
|
|
assert await asyncio.to_thread(entered.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
resume.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
rows = snapshot()
|
|
assert len(rows) == 1
|
|
row = next(iter(rows.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_lost_claim_response_recovers_exact_owner_as_uncertain(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
compare = engine.coordinator.store.compare_and_set
|
|
lose_background_reply = True
|
|
|
|
def commit_then_disconnect(kind, key, expected, value):
|
|
nonlocal lose_background_reply
|
|
result = compare(kind, key, expected, value)
|
|
document = json.loads(value)
|
|
if (result and lose_background_reply
|
|
and any(row.get("kind") == "background"
|
|
for row in document.get("claims", {}).values())):
|
|
lose_background_reply = False
|
|
raise RuntimeError("committed background claim reply lost")
|
|
return result
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unknown background admission dispatched host work")
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", commit_then_disconnect)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_unavailable"
|
|
|
|
before = engine.coordinator.snapshot()
|
|
assert len(before) == 1
|
|
row = next(iter(before.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["kind"] == "background"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
with pytest.raises(CoordinationAdmissionRejected) as blocked:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert blocked.value.error_code == "coordination_uncertain"
|
|
assert engine.coordinator.snapshot() == before
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_runtime_after_claim_commit_recovers_exact_owner_as_uncertain(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
claim = engine.coordinator.claim
|
|
|
|
def commit_then_raise(*args, **kwargs):
|
|
result = claim(*args, **kwargs)
|
|
if kwargs.get("kind") == "background":
|
|
raise RuntimeError("committed background claim reply lost")
|
|
return result
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unknown background admission dispatched host work")
|
|
|
|
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
with pytest.raises(RuntimeError, match="claim reply lost"):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
row = next(iter(rows.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["kind"] == "background"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["resources"] == []
|
|
assert row["metadata"] == identity(plan)
|
|
assert row["maintenance"] is False
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_recovered_claim_retain_retries_precommit_failure(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
claim = engine.coordinator.claim
|
|
retain = engine.coordinator.retain
|
|
retain_calls = 0
|
|
|
|
def commit_then_raise(*args, **kwargs):
|
|
result = claim(*args, **kwargs)
|
|
if kwargs.get("kind") == "background":
|
|
raise RuntimeError("committed background claim reply lost")
|
|
return result
|
|
|
|
def fail_first_retain(value, *args, **kwargs):
|
|
nonlocal retain_calls
|
|
kind = (value.get("kind") if isinstance(value, dict)
|
|
else value.kind)
|
|
if kind == "background":
|
|
retain_calls += 1
|
|
if retain_calls == 1:
|
|
raise CoordinationError("coordination_unavailable", 503)
|
|
return retain(value, *args, **kwargs)
|
|
|
|
async def forbidden(*_args, **_kwargs):
|
|
pytest.fail("unknown background admission dispatched host work")
|
|
|
|
monkeypatch.setattr(engine.coordinator, "claim", commit_then_raise)
|
|
monkeypatch.setattr(engine.coordinator, "retain", fail_first_retain)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", forbidden)
|
|
with pytest.raises(RuntimeError, match="claim reply lost"):
|
|
await vpc.reconcile("vpc-hk-1")
|
|
|
|
assert retain_calls == 2
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
row = next(iter(rows.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_cancel_during_final_snapshot_retains_root_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
body_completed = threading.Event()
|
|
snapshot_started = threading.Event()
|
|
release_snapshot = threading.Event()
|
|
snapshot = engine.coordinator.snapshot
|
|
|
|
async def host(*_args, **_kwargs):
|
|
body_completed.set()
|
|
|
|
def block_final_snapshot():
|
|
rows = snapshot()
|
|
if (body_completed.is_set() and len(rows) == 1
|
|
and next(iter(rows.values())).get("kind") == "background"):
|
|
snapshot_started.set()
|
|
assert release_snapshot.wait(5), "test did not release final snapshot"
|
|
return rows
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(engine.coordinator, "snapshot", block_final_snapshot)
|
|
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
|
|
try:
|
|
assert await asyncio.to_thread(snapshot_started.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
release_snapshot.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
rows = snapshot()
|
|
assert len(rows) == 1
|
|
row = next(iter(rows.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["kind"] == "background"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_cancel_after_committed_root_release_keeps_confirmed_settlement(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, _target = await seed(engine, store)
|
|
release = engine.coordinator.release
|
|
committed = threading.Event()
|
|
resume = threading.Event()
|
|
|
|
async def host(*_args, **_kwargs):
|
|
return None
|
|
|
|
def commit_then_block(target):
|
|
result = release(target)
|
|
kind = (target.get("kind") if isinstance(target, dict)
|
|
else target.kind)
|
|
if kind == "background" and result:
|
|
committed.set()
|
|
assert resume.wait(5), "test did not release committed root"
|
|
return result
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(engine.coordinator, "release", commit_then_block)
|
|
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
|
|
try:
|
|
assert await asyncio.to_thread(committed.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
resume.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
# The exact release CAS committed before cancellation. Recreating
|
|
# the deleted owner here would turn a confirmed settlement into a
|
|
# false uncertainty fence.
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_failed_final_release_falls_back_to_uncertain_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
compare = engine.coordinator.store.compare_and_set
|
|
fail_release = True
|
|
|
|
def disconnect_before_release(kind, key, expected, value):
|
|
nonlocal fail_release
|
|
before = json.loads(expected)
|
|
after = json.loads(value)
|
|
background = [row for row in before.get("claims", {}).values()
|
|
if row.get("kind") == "background"]
|
|
if fail_release and len(background) == 1 and not after.get("claims"):
|
|
fail_release = False
|
|
raise RuntimeError("background release did not commit")
|
|
return compare(kind, key, expected, value)
|
|
|
|
async def host(*_args, **_kwargs):
|
|
return None
|
|
|
|
monkeypatch.setattr(
|
|
engine.coordinator.store, "compare_and_set", disconnect_before_release)
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_unavailable"
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
row = next(iter(rows.values()))
|
|
assert row["state"] == "uncertain"
|
|
assert row["kind"] == "background"
|
|
assert row["operation_id"] == OPERATION_ID
|
|
assert row["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_active_child_leak_retains_root_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:leaked-child",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert rows[leaked.token]["state"] == "uncertain"
|
|
roots = [row for token, row in rows.items() if token != leaked.token]
|
|
assert len(roots) == 1
|
|
assert roots[0]["state"] == "uncertain"
|
|
assert roots[0]["kind"] == "background"
|
|
assert roots[0]["operation_id"] == OPERATION_ID
|
|
assert roots[0]["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_deterministic_failure_with_child_is_upgraded_to_uncertain(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:failed-child",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
raise VpcError("deterministic failure", 409,
|
|
"vpc_revision_conflict")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert all(row["state"] == "uncertain" for row in rows.values())
|
|
assert rows[leaked.token]["kind"] == "resource"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_child_retain_failure_retries_after_root_fence(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
child_retain_calls = 0
|
|
retain = engine.coordinator.retain
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:retry-child-retain",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
|
|
def fail_first_child_retain(target, *args, **kwargs):
|
|
nonlocal child_retain_calls
|
|
token = (target.token if hasattr(target, "token")
|
|
else target.get("token"))
|
|
if leaked is not None and token == leaked.token:
|
|
child_retain_calls += 1
|
|
if child_retain_calls == 1:
|
|
raise CoordinationError(
|
|
"coordination_unavailable", 503)
|
|
return retain(target, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "retain", fail_first_child_retain)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
assert child_retain_calls == 2
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert all(row["state"] == "uncertain" for row in rows.values())
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_cancel_during_child_retain_fences_all_children_and_root(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = []
|
|
blocked = False
|
|
entered = threading.Event()
|
|
resume = threading.Event()
|
|
retain = engine.coordinator.retain
|
|
|
|
async def host(*_args, **_kwargs):
|
|
context = current_execution()
|
|
for suffix in ("one", "two"):
|
|
leaked.append(engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=(f"test:cancel-child-retain-{suffix}",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
))
|
|
|
|
def commit_then_block(target, *args, **kwargs):
|
|
nonlocal blocked
|
|
token = (target.token if hasattr(target, "token")
|
|
else target.get("token"))
|
|
result = retain(target, *args, **kwargs)
|
|
if leaked and token == leaked[0].token and not blocked:
|
|
blocked = True
|
|
entered.set()
|
|
assert resume.wait(5), "test did not release child retain"
|
|
return result
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(engine.coordinator, "retain", commit_then_block)
|
|
task = asyncio.create_task(vpc.reconcile("vpc-hk-1"))
|
|
try:
|
|
assert await asyncio.to_thread(entered.wait, 5)
|
|
task.cancel()
|
|
finally:
|
|
resume.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 3
|
|
assert all(rows[item.token]["state"] == "uncertain"
|
|
for item in leaked)
|
|
assert all(row["state"] == "uncertain" for row in rows.values())
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_periodic_unknown_execution_never_returns_success(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
|
|
async def host(*_args, **_kwargs):
|
|
mark_execution_unknown()
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
await vpc.reconcile("vpc-hk-1")
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
rows = engine.coordinator.snapshot()
|
|
assert_retained_claim_family(
|
|
rows, plan, root_kind="background",
|
|
member_uuids=(INSTANCE_A, INSTANCE_B))
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_active_child_leak_retains_root_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:http-leaked-child",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert rows[leaked.token]["state"] == "uncertain"
|
|
roots = [row for token, row in rows.items() if token != leaked.token]
|
|
assert len(roots) == 1
|
|
assert roots[0]["state"] == "uncertain"
|
|
assert roots[0]["kind"] == "request"
|
|
assert roots[0]["operation_id"] == OPERATION_ID
|
|
assert roots[0]["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_child_retain_failure_still_retains_root_owner(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
child_retain_calls = 0
|
|
retain = engine.coordinator.retain
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:http-child-retain-failure",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
|
|
def fail_child_retain(claim, *args, **kwargs):
|
|
nonlocal child_retain_calls
|
|
token = (claim.token if hasattr(claim, "token") else
|
|
claim.get("token"))
|
|
if leaked is not None and token == leaked.token:
|
|
child_retain_calls += 1
|
|
if child_retain_calls == 1:
|
|
raise CoordinationError(
|
|
"coordination_unavailable", 503)
|
|
return retain(claim, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(engine.coordinator, "retain", fail_child_retain)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert child_retain_calls == 2
|
|
assert rows[leaked.token]["state"] == "uncertain"
|
|
roots = [row for token, row in rows.items() if token != leaked.token]
|
|
assert len(roots) == 1
|
|
assert roots[0]["state"] == "uncertain"
|
|
assert roots[0]["operation_id"] == OPERATION_ID
|
|
assert roots[0]["metadata"] == identity(plan)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_unknown_execution_without_residual_returns_503(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
|
|
async def host(*_args, **_kwargs):
|
|
mark_execution_unknown()
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
rows = engine.coordinator.snapshot()
|
|
assert_retained_claim_family(
|
|
rows, plan, root_kind="request",
|
|
member_uuids=(INSTANCE_A, INSTANCE_B))
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_lost_child_release_reply_with_absent_token_returns_503(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
release = engine.coordinator.release
|
|
lost_reply = False
|
|
|
|
async def host(*_args, **_kwargs):
|
|
return None
|
|
|
|
def commit_child_release_then_disconnect(claim):
|
|
nonlocal lost_reply
|
|
released = release(claim)
|
|
if claim.kind == "resource" and not lost_reply:
|
|
lost_reply = True
|
|
mark_execution_unknown()
|
|
raise CoordinationError(
|
|
"coordination_unavailable", 503)
|
|
return released
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "release",
|
|
commit_child_release_then_disconnect)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert lost_reply is True
|
|
rows = engine.coordinator.snapshot()
|
|
assert_retained_claim_family(
|
|
rows, plan, root_kind="request",
|
|
member_uuids=(INSTANCE_A,))
|
|
assert all(
|
|
f"vm:host-a:{INSTANCE_B}" not in row["resources"]
|
|
for row in rows.values())
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_exception_retains_residual_child_and_root(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:http-exception-child",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
raise VpcError("deterministic failure", 409,
|
|
"vpc_revision_conflict")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 2
|
|
assert rows[leaked.token]["state"] == "uncertain"
|
|
roots = [row for token, row in rows.items()
|
|
if token != leaked.token]
|
|
assert len(roots) == 1
|
|
assert roots[0]["state"] == "uncertain"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_unhandled_exception_retains_residual_child_and_root(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, target = await seed(engine, store)
|
|
plan = parse_vpc_plan(target)
|
|
leaked = None
|
|
|
|
async def host(*_args, **_kwargs):
|
|
nonlocal leaked
|
|
context = current_execution()
|
|
leaked = engine.coordinator.claim(
|
|
operation_id=OPERATION_ID,
|
|
resources=("test:http-unhandled-exception-child",),
|
|
execution_id=context.claim.execution_id,
|
|
kind="resource",
|
|
metadata=identity(plan),
|
|
)
|
|
raise RuntimeError("unhandled handler failure")
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
rows = engine.coordinator.snapshot()
|
|
# The exception escapes before the nested catalog/member locks can
|
|
# finish, so every claim from this execution must be fenced.
|
|
assert len(rows) == 5
|
|
assert rows[leaked.token]["state"] == "uncertain"
|
|
assert all(row["state"] == "uncertain"
|
|
and row["operation_id"] == OPERATION_ID
|
|
and row["metadata"] == identity(plan)
|
|
for row in rows.values())
|
|
roots = [row for row in rows.values()
|
|
if row["kind"] == "request"]
|
|
assert len(roots) == 1
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_false_root_release_is_read_back_and_returns_503(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, _target = await seed(engine, store)
|
|
release = engine.coordinator.release
|
|
failed = False
|
|
|
|
async def host(*_args, **_kwargs):
|
|
return None
|
|
|
|
def false_root_release(claim):
|
|
nonlocal failed
|
|
kind = (claim.kind if hasattr(claim, "kind") else
|
|
claim.get("kind"))
|
|
if kind == "request" and not failed:
|
|
failed = True
|
|
return False
|
|
return release(claim)
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "release", false_root_release)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert failed is True
|
|
rows = engine.coordinator.snapshot()
|
|
assert len(rows) == 1
|
|
root = next(iter(rows.values()))
|
|
assert root["state"] == "uncertain"
|
|
assert root["operation_id"] == OPERATION_ID
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_http_lost_root_release_reply_reads_absence_and_returns_503(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
vpc, _target = await seed(engine, store)
|
|
release = engine.coordinator.release
|
|
lost_reply = False
|
|
|
|
async def host(*_args, **_kwargs):
|
|
return None
|
|
|
|
def commit_root_release_then_disconnect(claim):
|
|
nonlocal lost_reply
|
|
released = release(claim)
|
|
kind = (claim.kind if hasattr(claim, "kind") else
|
|
claim.get("kind"))
|
|
if kind == "request" and not lost_reply:
|
|
lost_reply = True
|
|
raise CoordinationError(
|
|
"coordination_unavailable", 503)
|
|
return released
|
|
|
|
monkeypatch.setattr(vpc, "_reconcile_dataplane_unlocked", host)
|
|
monkeypatch.setattr(
|
|
engine.coordinator, "release",
|
|
commit_root_release_then_disconnect)
|
|
status, result = await request(
|
|
engine, "POST", "/internal/v1/vpcs/vpc-hk-1/reconcile", {})
|
|
assert status == 503, result
|
|
assert result["error_code"] == "coordination_uncertain"
|
|
assert lost_reply is True
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|