fix: recover stopped Fabric operations before new reconciliation
parent
d50e6d529e
commit
0c8cf9d853
@ -0,0 +1,297 @@
|
||||
"""Real ASGI Fabric rejection receipts must not accumulate empty claims."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_coordination import local_backends, request, running_engine # noqa: F401
|
||||
from test_fabric import plan_payload
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationClaimRejected, CoordinationError, DurableCoordinator,
|
||||
current_execution, mark_execution_unknown, mark_remote_write_started,
|
||||
)
|
||||
|
||||
|
||||
OLD_OPERATION = "a" * 32
|
||||
TARGETS = [
|
||||
("apply", "hk-fabric"),
|
||||
("reconcile", "hk-fabric"),
|
||||
("reconcile", "hk.fabric-1"),
|
||||
]
|
||||
|
||||
|
||||
def target(action, fabric_id):
|
||||
if action == "apply":
|
||||
value = plan_payload()
|
||||
value["fabric_id"] = fabric_id
|
||||
return "/internal/v1/fabric/apply", value
|
||||
return f"/internal/v1/fabric/{fabric_id}/reconcile", {}
|
||||
|
||||
|
||||
def old_owner(store, fabric_id, state="uncertain"):
|
||||
owner = DurableCoordinator(store)
|
||||
claim = owner.claim(
|
||||
operation_id=OLD_OPERATION,
|
||||
resources=("catalog:network", f"fabric:{fabric_id}"), kind="background",
|
||||
metadata={"job_kind": "fabric_operation", "job_id": OLD_OPERATION})
|
||||
if state == "uncertain":
|
||||
assert owner.retain(claim)
|
||||
return claim, owner.snapshot()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action,fabric_id", TARGETS)
|
||||
@pytest.mark.parametrize("blocking_state", ["active", "uncertain"])
|
||||
def test_real_catalog_rejection_returns_409_without_new_request_claim(
|
||||
tmp_path, monkeypatch, action, fabric_id, blocking_state):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, services):
|
||||
original, before = old_owner(store, fabric_id, blocking_state)
|
||||
remote_calls = []
|
||||
|
||||
async def forbidden_components(*_args, **_kwargs):
|
||||
remote_calls.append(True)
|
||||
pytest.fail("rejected Fabric admission reached a host")
|
||||
|
||||
monkeypatch.setattr(engine.fabric, "_components", forbidden_components)
|
||||
path, value = target(action, fabric_id)
|
||||
code = ("coordination_uncertain" if blocking_state == "uncertain"
|
||||
else "coordination_busy")
|
||||
# Deliberately omit actor.operation_id, matching browser Fabric
|
||||
# actions. Each attempt used to leave a new resources=[] claim.
|
||||
for _attempt in range(3):
|
||||
status, body = await request(engine, "POST", path, value)
|
||||
assert status == 409, body
|
||||
assert body == {"error": str(CoordinationError(code)), "error_code": code}
|
||||
assert engine.coordinator.snapshot() == before
|
||||
assert engine.coordinator.operation_claim(OLD_OPERATION) == before[original.token]
|
||||
assert store.list("fabric_desired") == {}
|
||||
assert store.list("fabric_node") == {}
|
||||
assert store.list("operation") == {}
|
||||
assert remote_calls == []
|
||||
assert services == {}
|
||||
assert engine.fabric._operation_tasks == set()
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocking_state", ["active", "uncertain"])
|
||||
def test_initial_request_claim_rejection_is_definitive_without_an_execution_context(
|
||||
tmp_path, monkeypatch, blocking_state):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
blocker = DurableCoordinator(store)
|
||||
original = blocker.claim(maintenance=True, kind="resource")
|
||||
if blocking_state == "uncertain":
|
||||
assert blocker.retain(original)
|
||||
before = blocker.snapshot()
|
||||
calls = []
|
||||
|
||||
async def forbidden(*_args, **_kwargs):
|
||||
calls.append(True)
|
||||
pytest.fail("globally rejected request reached the Fabric handler")
|
||||
|
||||
monkeypatch.setattr(engine.fabric, "enqueue_reconcile", forbidden)
|
||||
status, body = await request(
|
||||
engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {})
|
||||
code = ("coordination_uncertain" if blocking_state == "uncertain"
|
||||
else "coordination_busy")
|
||||
assert status == 409, body
|
||||
assert body == {"error": str(CoordinationError(code)), "error_code": code}
|
||||
assert blocker.snapshot() == before
|
||||
assert calls == []
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", ["coordination_busy", "coordination_uncertain"])
|
||||
def test_initial_generic_claim_error_is_not_a_definitive_rejection(
|
||||
tmp_path, monkeypatch, code):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
_original, before = old_owner(store, "hk.fabric-1")
|
||||
claim = engine.coordinator.claim
|
||||
attempts = []
|
||||
|
||||
def unproven_claim(*args, **kwargs):
|
||||
if kwargs.get("kind") == "request":
|
||||
attempts.append(True)
|
||||
raise CoordinationError(code, 503)
|
||||
return claim(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(engine.coordinator, "claim", unproven_claim)
|
||||
status, body = await request(
|
||||
engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {})
|
||||
assert status == 503, body
|
||||
assert body["error_code"] == code
|
||||
assert attempts == [True]
|
||||
assert engine.coordinator.snapshot() == before
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
class LookalikeClaimRejected(CoordinationClaimRejected):
|
||||
"""A subclass must not inherit the exact rejection receipt authority."""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action,fabric_id", TARGETS)
|
||||
@pytest.mark.parametrize("failure", [
|
||||
"generic_busy", "generic_uncertain", "remote_write", "execution_unknown",
|
||||
"background_child", "subclass", "unreviewed_code",
|
||||
])
|
||||
def test_unproven_fabric_failure_remains_503_and_keeps_request_ownership(
|
||||
tmp_path, monkeypatch, action, fabric_id, failure):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
original, before = old_owner(store, fabric_id)
|
||||
seen = {}
|
||||
|
||||
async def fail(*_args, **_kwargs):
|
||||
context = current_execution()
|
||||
seen["request"] = context.claim
|
||||
assert context.claim.resources == ()
|
||||
assert context.claim.operation_id is None
|
||||
if failure == "remote_write":
|
||||
mark_remote_write_started()
|
||||
elif failure == "execution_unknown":
|
||||
mark_execution_unknown()
|
||||
elif failure == "background_child":
|
||||
child = context.coordinator.claim(
|
||||
operation_id="b" * 32, resources=("fabric:other",),
|
||||
execution_id=context.claim.execution_id, kind="background",
|
||||
metadata={"job_kind": "fabric_operation", "job_id": "b" * 32})
|
||||
seen["child"] = context.coordinator.snapshot()[child.token]
|
||||
if failure in {"generic_busy", "generic_uncertain"}:
|
||||
code = ("coordination_busy" if failure == "generic_busy"
|
||||
else "coordination_uncertain")
|
||||
raise CoordinationError(code, 503)
|
||||
if failure == "subclass":
|
||||
raise LookalikeClaimRejected("coordination_uncertain", 503)
|
||||
if failure == "unreviewed_code":
|
||||
raise CoordinationClaimRejected("coordination_unavailable", 503)
|
||||
raise CoordinationClaimRejected("coordination_uncertain", 503)
|
||||
|
||||
monkeypatch.setattr(engine.fabric, "enqueue_" + action, fail)
|
||||
path, value = target(action, fabric_id)
|
||||
status, body = await request(engine, "POST", path, value)
|
||||
expected_code = ("coordination_busy" if failure == "generic_busy"
|
||||
else "coordination_unavailable" if failure == "unreviewed_code"
|
||||
else "coordination_uncertain")
|
||||
assert status == 503, body
|
||||
assert body["error_code"] == expected_code
|
||||
claims = engine.coordinator.snapshot()
|
||||
assert claims[original.token] == before[original.token]
|
||||
rejected = claims[seen["request"].token]
|
||||
assert rejected["state"] == "uncertain"
|
||||
assert rejected["resources"] == []
|
||||
assert rejected["operation_id"] is None
|
||||
expected_tokens = {original.token, seen["request"].token}
|
||||
if failure == "background_child":
|
||||
child = seen["child"]
|
||||
expected_tokens.add(child["token"])
|
||||
assert claims[child["token"]] == child
|
||||
assert set(claims) == expected_tokens
|
||||
assert store.list("fabric_desired") == {}
|
||||
assert store.list("operation") == {}
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_claim_rejection_on_fabric_probe_does_not_get_mutation_receipt(
|
||||
tmp_path, monkeypatch):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
original, before = old_owner(store, "hk-fabric")
|
||||
seen = {}
|
||||
|
||||
async def fail(*_args, **_kwargs):
|
||||
seen["request"] = current_execution().claim
|
||||
raise CoordinationClaimRejected("coordination_uncertain", 503)
|
||||
|
||||
monkeypatch.setattr(engine.fabric, "probe_hosts", fail)
|
||||
status, body = await request(
|
||||
engine, "POST", "/internal/v1/fabric/probe", {"host_ids": []})
|
||||
assert status == 503, body
|
||||
assert body["error_code"] == "coordination_uncertain"
|
||||
claims = engine.coordinator.snapshot()
|
||||
assert claims[original.token] == before[original.token]
|
||||
assert set(claims) == {original.token, seen["request"].token}
|
||||
assert claims[seen["request"].token]["state"] == "uncertain"
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fault", [
|
||||
"snapshot_unavailable", "release_unavailable", "release_refused", "release_reply_lost",
|
||||
])
|
||||
def test_rejection_receipt_requires_confirmed_snapshot_and_request_release(
|
||||
tmp_path, monkeypatch, fault):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
original, before = old_owner(store, "hk.fabric-1")
|
||||
seen = {"injected": False, "rejected": False}
|
||||
snapshot = engine.coordinator.snapshot
|
||||
release = engine.coordinator.release
|
||||
|
||||
async def fail(*_args, **_kwargs):
|
||||
seen["request"] = current_execution().claim
|
||||
seen["rejected"] = True
|
||||
raise CoordinationClaimRejected("coordination_uncertain", 503)
|
||||
|
||||
def unavailable_snapshot():
|
||||
if fault == "snapshot_unavailable" and seen["rejected"] and not seen["injected"]:
|
||||
seen["injected"] = True
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
return snapshot()
|
||||
|
||||
def uncertain_release(claim):
|
||||
if (fault != "snapshot_unavailable" and seen["rejected"]
|
||||
and claim.token == seen["request"].token and not seen["injected"]):
|
||||
seen["injected"] = True
|
||||
if fault == "release_refused":
|
||||
# Another owner retained this exact claim before release.
|
||||
assert engine.coordinator.retain(claim)
|
||||
return False
|
||||
if fault == "release_reply_lost":
|
||||
assert release(claim)
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
return release(claim)
|
||||
|
||||
monkeypatch.setattr(engine.fabric, "enqueue_reconcile", fail)
|
||||
monkeypatch.setattr(engine.coordinator, "snapshot", unavailable_snapshot)
|
||||
monkeypatch.setattr(engine.coordinator, "release", uncertain_release)
|
||||
status, body = await request(
|
||||
engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {})
|
||||
assert seen["injected"] is True
|
||||
assert status == 503, body
|
||||
assert body["error_code"] == (
|
||||
"coordination_uncertain" if fault == "release_refused" else "coordination_unavailable")
|
||||
claims = snapshot()
|
||||
assert claims[original.token] == before[original.token]
|
||||
# A lost release reply may have removed this empty, unstarted
|
||||
# request. It must never remove or alter the original resource fence.
|
||||
if fault == "release_reply_lost":
|
||||
assert claims == before
|
||||
else:
|
||||
assert set(claims) == {original.token, seen["request"].token}
|
||||
if fault == "release_refused":
|
||||
assert claims[seen["request"].token]["state"] == "uncertain"
|
||||
assert store.list("fabric_desired") == {}
|
||||
assert store.list("operation") == {}
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method,path", [
|
||||
("PUT", "/internal/v1/fabric/apply"),
|
||||
("POST", "/internal/v1/fabric/hk.fabric-1/reconcile/extra"),
|
||||
])
|
||||
def test_similar_method_or_route_does_not_expand_fabric_admission_receipt(
|
||||
tmp_path, method, path):
|
||||
async def exercise():
|
||||
async with running_engine(tmp_path) as (engine, store, _services):
|
||||
_original, before = old_owner(store, "hk-fabric")
|
||||
status, _body = await request(engine, method, path, {})
|
||||
assert status == 404
|
||||
assert engine.coordinator.snapshot() == before
|
||||
|
||||
asyncio.run(exercise())
|
||||
@ -0,0 +1,451 @@
|
||||
"""Explicit recovery proves the exact old host jobs stopped before claim handoff."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationClaimRejected, CoordinationError, DurableCoordinator,
|
||||
current_execution, execution_context,
|
||||
)
|
||||
from virsh_panel.fabric import (
|
||||
FabricCoordinator, FabricError, FabricHostAdapter,
|
||||
_fabric_operation_terminal_proof_script,
|
||||
)
|
||||
from virsh_panel.internal import _DurableAsyncLock
|
||||
from test_durable_fabric_jobs import FabricStore
|
||||
from test_fabric import (
|
||||
AsyncCoordinatorAdapter, Registry, _active_fabric_record, _async_components,
|
||||
_reconcile_observation, _remote_operation_status, plan,
|
||||
)
|
||||
|
||||
OLD_ID = "a" * 32
|
||||
SCOPE = ["catalog:network", "fabric:hk-fabric"]
|
||||
|
||||
|
||||
class RecoveryAdapter(AsyncCoordinatorAdapter):
|
||||
def terminal_operation_proof(self, operation_id, digest):
|
||||
assert operation_id == OLD_ID and digest == plan().digest
|
||||
self.manager.proof_calls += 1
|
||||
if self.manager.proof_error:
|
||||
raise self.manager.proof_error
|
||||
return _remote_operation_status(operation_id, self.manager.proof_status)
|
||||
|
||||
def observe(self, current, host, keys):
|
||||
result = super().observe(current, host, keys)
|
||||
if self.manager.async_dispatch_calls == 0 and self.manager.old_readback_mismatch:
|
||||
result.update(state="degraded", config_ok=False, transport_verified=False)
|
||||
return result
|
||||
|
||||
|
||||
def retained_fixture(monkeypatch, *, store=None):
|
||||
current = plan()
|
||||
store = store or FabricStore()
|
||||
owner = DurableCoordinator(store)
|
||||
catalog = _DurableAsyncLock(owner, ("catalog:network",))
|
||||
components = _async_components(current)
|
||||
for component in components.values():
|
||||
component.manager.proof_calls = 0
|
||||
component.manager.proof_status = "succeeded"
|
||||
component.manager.proof_error = None
|
||||
component.manager.old_readback_mismatch = True
|
||||
fabric = FabricCoordinator(store, Registry(components), catalog_lock=lambda: catalog)
|
||||
desired = _active_fabric_record(current)
|
||||
desired["operation"] = fabric._new_root_operation(OLD_ID, "reconcile", 100)
|
||||
desired["operation"].update(status="uncertain", started_at=101, finished_at=103)
|
||||
for node in desired["nodes"]:
|
||||
node["operation"] = fabric._new_node_operation(OLD_ID, "apply", 100)
|
||||
node["operation"].update(status="uncertain", started_at=101, finished_at=103)
|
||||
fabric._invalidate(desired)
|
||||
fabric._save("fabric_desired", current.fabric_id, desired)
|
||||
previous_owner = DurableCoordinator(store)
|
||||
old = previous_owner.claim(
|
||||
operation_id=OLD_ID, resources=SCOPE, kind="background",
|
||||
metadata={"job_kind": "fabric_operation", "job_id": OLD_ID})
|
||||
assert previous_owner.retain(old)
|
||||
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", RecoveryAdapter)
|
||||
monkeypatch.setattr("virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
||||
return store, owner, fabric, components, old
|
||||
|
||||
|
||||
@pytest.mark.parametrize("old_status", ["succeeded", "failed"])
|
||||
@pytest.mark.parametrize("mismatch", [False, True])
|
||||
def test_explicit_terminal_recovery_closes_old_receipt_and_transfers_new_job(
|
||||
monkeypatch, old_status, mismatch):
|
||||
async def run():
|
||||
store, owner, fabric, components, old = retained_fixture(monkeypatch)
|
||||
for component in components.values():
|
||||
component.manager.proof_status = old_status
|
||||
component.manager.old_readback_mismatch = mismatch
|
||||
parent = owner.claim(kind="request")
|
||||
try:
|
||||
with execution_context(owner, parent):
|
||||
result = await fabric.enqueue_reconcile("hk-fabric")
|
||||
assert result["operation"]["id"] != OLD_ID
|
||||
current = fabric._load("fabric_desired", "hk-fabric")
|
||||
receipt = current["reconciled_operation"]
|
||||
assert receipt["operation"]["id"] == OLD_ID
|
||||
assert receipt["operation"]["status"] == "failed"
|
||||
assert all(node["remote_status"] == old_status for node in receipt["nodes"])
|
||||
assert all(node["operation"]["status"] == receipt["operation"]["status"]
|
||||
for node in receipt["nodes"])
|
||||
assert receipt["claim_token"] == old.token
|
||||
row = json.loads(store.get("coordination", "admission-v1"))
|
||||
assert len(row["resolutions"]) == 1
|
||||
assert row["resolutions"][0]["operation_id"] == OLD_ID
|
||||
assert old.token not in row["claims"]
|
||||
children = [claim for claim in row["claims"].values() if claim["kind"] == "background"]
|
||||
assert len(children) == 1
|
||||
assert children[0]["operation_id"] == result["operation"]["id"]
|
||||
with pytest.raises(CoordinationClaimRejected):
|
||||
DurableCoordinator(store).claim(maintenance=True, kind="update")
|
||||
assert owner.release(parent)
|
||||
await asyncio.wait_for(asyncio.gather(*tuple(fabric._operation_tasks)), 2)
|
||||
assert owner.snapshot() == {}
|
||||
assert all(component.manager.proof_calls == 1 for component in components.values())
|
||||
assert all(component.manager.async_dispatch_calls == int(mismatch)
|
||||
for component in components.values())
|
||||
finally:
|
||||
await fabric.shutdown_operations()
|
||||
fabric.close()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", [
|
||||
"active", "other_operation", "extra_resource", "wrong_kind", "wrong_job", "maintenance",
|
||||
"extra_claim", "node_id", "node_phase", "node_checkpoint", "node_pending", "root_digest",
|
||||
"root_kind", "root_status", "root_times", "host_profile",
|
||||
"running_proof", "queued_proof", "uncertain_proof", "proof_failure",
|
||||
])
|
||||
def test_unproven_or_different_owner_is_never_resolved(monkeypatch, mutation):
|
||||
async def run():
|
||||
store, owner, fabric, components, old = retained_fixture(monkeypatch)
|
||||
doc = json.loads(store.get("coordination", "admission-v1"))
|
||||
claim = doc["claims"][old.token]
|
||||
if mutation == "active": claim["state"] = "active"
|
||||
elif mutation == "other_operation": claim["operation_id"] = "b" * 32
|
||||
elif mutation == "extra_resource": claim["resources"].append("host:host-a")
|
||||
elif mutation == "wrong_kind": claim["kind"] = "resource"
|
||||
elif mutation == "wrong_job": claim["metadata"]["job_kind"] = "other_job"
|
||||
elif mutation == "maintenance": claim["maintenance"] = True
|
||||
elif mutation == "extra_claim":
|
||||
extra = {**claim, "token": "d" * 64, "execution_id": "e" * 64}
|
||||
doc["claims"][extra["token"]] = extra
|
||||
store.set("coordination", "admission-v1", json.dumps(doc))
|
||||
desired = fabric._load("fabric_desired", "hk-fabric")
|
||||
node = desired["nodes"][1]["operation"]
|
||||
if mutation == "node_id": node["id"] = "b" * 32
|
||||
elif mutation == "node_phase": node["phase"] = "unknown"
|
||||
elif mutation == "node_checkpoint": node["checkpoint"] = "unknown"
|
||||
elif mutation == "node_pending": node.update(phase="identity", checkpoint="dispatch", status="pending")
|
||||
elif mutation == "root_digest": desired["plan_digest"] = "0" * 64
|
||||
elif mutation == "root_kind": desired["operation"]["kind"] = "other"
|
||||
elif mutation == "root_status": desired["operation"]["status"] = "other"
|
||||
elif mutation == "root_times": desired["operation"]["created_at"] = True
|
||||
elif mutation == "host_profile": components["host-b"].manager.profile["host_id"] = "host-c"
|
||||
fabric._save("fabric_desired", "hk-fabric", desired)
|
||||
manager = components["host-b"].manager
|
||||
if mutation.endswith("_proof"): manager.proof_status = mutation[:-6]
|
||||
elif mutation == "proof_failure":
|
||||
manager.proof_error = FabricError("unsafe private stderr", 502, "fabric_operation_status_invalid")
|
||||
# Maintenance admission itself rejects the new request before it exists.
|
||||
if mutation == "maintenance":
|
||||
with pytest.raises(CoordinationClaimRejected): owner.claim(kind="request")
|
||||
assert owner.snapshot()[old.token]["state"] == "uncertain"
|
||||
fabric.close()
|
||||
return
|
||||
parent = owner.claim(kind="request")
|
||||
before = store.get("fabric_desired", "hk-fabric")
|
||||
try:
|
||||
with execution_context(owner, parent), pytest.raises((FabricError, CoordinationError)):
|
||||
await fabric.enqueue_reconcile("hk-fabric")
|
||||
assert store.get("fabric_desired", "hk-fabric") == before
|
||||
assert old.token in owner.snapshot()
|
||||
assert json.loads(store.get("coordination", "admission-v1"))["resolutions"] == []
|
||||
assert not fabric._operation_tasks
|
||||
assert all(component.manager.async_dispatch_calls == 0 for component in components.values())
|
||||
finally:
|
||||
owner.release(parent)
|
||||
fabric.close()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_periodic_observation_keeps_retained_claim_and_does_not_probe_or_dispatch(monkeypatch):
|
||||
async def run():
|
||||
store, owner, fabric, components, old = retained_fixture(monkeypatch)
|
||||
before = store.get("fabric_desired", "hk-fabric")
|
||||
await fabric.reconcile_all()
|
||||
assert store.get("fabric_desired", "hk-fabric") == before
|
||||
assert list(owner.snapshot()) == [old.token]
|
||||
assert all(component.manager.proof_calls == 0 and component.manager.async_dispatch_calls == 0
|
||||
for component in components.values())
|
||||
fabric.close()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["metadata", "updated_at", "resources"])
|
||||
def test_atomic_handoff_rejects_stale_claim_snapshot_without_resolution(change):
|
||||
store = FabricStore()
|
||||
old_owner, new_owner = DurableCoordinator(store), DurableCoordinator(store)
|
||||
old = old_owner.claim(operation_id=OLD_ID, resources=SCOPE, kind="background")
|
||||
old_owner.retain(old)
|
||||
snapshot = new_owner.snapshot()[old.token]
|
||||
if change == "metadata": old_owner.update_metadata(old, {"later": True})
|
||||
elif change == "updated_at":
|
||||
doc = json.loads(store.get("coordination", "admission-v1"))
|
||||
doc["claims"][old.token]["updated_at"] += 1
|
||||
store.set("coordination", "admission-v1", json.dumps(doc))
|
||||
else:
|
||||
doc = json.loads(store.get("coordination", "admission-v1"))
|
||||
doc["claims"][old.token]["resources"].append("host:host-a")
|
||||
store.set("coordination", "admission-v1", json.dumps(doc))
|
||||
with pytest.raises(CoordinationClaimRejected):
|
||||
new_owner.resolve_and_claim(snapshot, "terminal proof", execution_id="f" * 64)
|
||||
doc = json.loads(store.get("coordination", "admission-v1"))
|
||||
assert old.token in doc["claims"] and doc["resolutions"] == []
|
||||
|
||||
|
||||
def test_every_atomic_commit_preserves_fence_and_failed_handoff_stays_exclusive(monkeypatch):
|
||||
class InspectStore(FabricStore):
|
||||
inspect = False
|
||||
def compare_and_set(self, kind, key, expected, desired):
|
||||
if self.inspect and kind == "coordination":
|
||||
claims = json.loads(desired)["claims"].values()
|
||||
assert any(set(SCOPE) <= set(row["resources"]) for row in claims)
|
||||
return super().compare_and_set(kind, key, expected, desired)
|
||||
async def run():
|
||||
store = InspectStore()
|
||||
store, owner, fabric, _components, old = retained_fixture(monkeypatch, store=store)
|
||||
parent = owner.claim(kind="request")
|
||||
store.inspect = True
|
||||
original_save = fabric._save
|
||||
def fail_terminal(kind, key, value):
|
||||
if kind == "fabric_desired" and "reconciled_operation" in value:
|
||||
raise RuntimeError("injected persistence failure")
|
||||
return original_save(kind, key, value)
|
||||
monkeypatch.setattr(fabric, "_save", fail_terminal)
|
||||
try:
|
||||
with execution_context(owner, parent), pytest.raises(RuntimeError):
|
||||
await fabric.enqueue_reconcile("hk-fabric")
|
||||
assert old.token not in owner.snapshot()
|
||||
replacement = next(row for row in owner.snapshot().values() if row["resources"] == SCOPE)
|
||||
assert replacement["state"] == "uncertain" and replacement["resources"] == SCOPE
|
||||
with pytest.raises(CoordinationClaimRejected):
|
||||
DurableCoordinator(store).claim(resources=("catalog:network",))
|
||||
assert fabric._load("fabric_desired", "hk-fabric")["operation"]["id"] == OLD_ID
|
||||
finally:
|
||||
owner.release(parent)
|
||||
fabric.close()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_lost_atomic_handoff_response_keeps_replacement_fence(monkeypatch):
|
||||
class LostReplyStore(FabricStore):
|
||||
lose_reply = False
|
||||
|
||||
def compare_and_set(self, kind, key, expected, desired):
|
||||
committed = super().compare_and_set(kind, key, expected, desired)
|
||||
if committed and self.lose_reply and kind == "coordination" and json.loads(desired)["resolutions"]:
|
||||
self.lose_reply = False
|
||||
raise RuntimeError("CAS committed, reply lost")
|
||||
return committed
|
||||
|
||||
async def run():
|
||||
store, owner, fabric, _components, old = retained_fixture(monkeypatch, store=LostReplyStore())
|
||||
parent = owner.claim(kind="request")
|
||||
store.lose_reply = True
|
||||
try:
|
||||
with execution_context(owner, parent), pytest.raises(CoordinationError) as raised:
|
||||
await fabric.enqueue_reconcile("hk-fabric")
|
||||
assert raised.value.error_code == "coordination_unavailable"
|
||||
rows = owner.snapshot()
|
||||
assert old.token not in rows
|
||||
assert any(row["resources"] == SCOPE for row in rows.values())
|
||||
assert fabric._load("fabric_desired", "hk-fabric")["operation"]["id"] == OLD_ID
|
||||
assert not fabric._operation_tasks
|
||||
with pytest.raises(CoordinationClaimRejected):
|
||||
DurableCoordinator(store).claim(maintenance=True, kind="update")
|
||||
finally:
|
||||
owner.release(parent)
|
||||
fabric.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["proof", "handoff", "new_owner"])
|
||||
def test_cancellation_cannot_release_fence_before_host_proof_or_new_owner(monkeypatch, phase):
|
||||
async def run():
|
||||
store, owner, fabric, components, old = retained_fixture(monkeypatch)
|
||||
entered, resume = threading.Event(), threading.Event()
|
||||
original_proof = RecoveryAdapter.terminal_operation_proof
|
||||
original_handoff = owner.resolve_and_claim
|
||||
original_claim = owner.claim
|
||||
def block_proof(adapter, operation_id, digest):
|
||||
if phase == "proof" and adapter.manager.host_id == "host-a":
|
||||
entered.set(); assert resume.wait(3)
|
||||
return original_proof(adapter, operation_id, digest)
|
||||
def block_handoff(*args, **kwargs):
|
||||
result = original_handoff(*args, **kwargs)
|
||||
if phase == "handoff": entered.set(); assert resume.wait(3)
|
||||
return result
|
||||
def block_child(**kwargs):
|
||||
result = original_claim(**kwargs)
|
||||
if phase == "new_owner" and kwargs.get("kind") == "background":
|
||||
entered.set(); assert resume.wait(3)
|
||||
return result
|
||||
monkeypatch.setattr(RecoveryAdapter, "terminal_operation_proof", block_proof)
|
||||
monkeypatch.setattr(owner, "resolve_and_claim", block_handoff)
|
||||
monkeypatch.setattr(owner, "claim", block_child)
|
||||
parent = owner.claim(kind="request")
|
||||
with execution_context(owner, parent): task = asyncio.create_task(fabric.enqueue_reconcile("hk-fabric"))
|
||||
try:
|
||||
for _ in range(300):
|
||||
if entered.is_set(): break
|
||||
await asyncio.sleep(.002)
|
||||
assert entered.is_set()
|
||||
task.cancel()
|
||||
await asyncio.sleep(.01)
|
||||
with pytest.raises(CoordinationClaimRejected):
|
||||
DurableCoordinator(store).claim(maintenance=True, kind="update")
|
||||
resume.set()
|
||||
with pytest.raises(asyncio.CancelledError): await task
|
||||
if phase == "proof":
|
||||
assert old.token in owner.snapshot()
|
||||
assert not fabric._operation_tasks
|
||||
else:
|
||||
assert old.token not in owner.snapshot()
|
||||
await asyncio.gather(*tuple(fabric._operation_tasks))
|
||||
assert not any(row["kind"] == "resource" for row in owner.snapshot().values())
|
||||
owner.release(parent)
|
||||
finally:
|
||||
resume.set()
|
||||
await fabric.shutdown_operations()
|
||||
fabric.close()
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def _proof_shell_fixture(tmp_path, *, status="succeeded", unit=None, drift=None):
|
||||
base = tmp_path / "virt-cloud"
|
||||
root = base / "fabric-operations"
|
||||
operation = root / OLD_ID
|
||||
operation.mkdir(parents=True)
|
||||
for path in (base, root, operation): path.chmod(0o700)
|
||||
digests = []
|
||||
for name in ("apply", "update", "runner"):
|
||||
path = operation / name
|
||||
path.write_text(f"#!/bin/sh\n# {name}\nexit 0\n")
|
||||
path.chmod(0o700)
|
||||
digests.append(hashlib.sha256(path.read_bytes()).hexdigest())
|
||||
manifest = operation / "manifest"
|
||||
manifest.write_text("\n".join(("schema=1", f"operation_id={OLD_ID}",
|
||||
f"plan_digest={plan().digest}", f"script_digest={digests[0]}",
|
||||
f"update_digest={digests[1]}", f"runner_digest={digests[2]}", "created_at=100", "")))
|
||||
manifest.chmod(0o600)
|
||||
receipt = operation / "status"
|
||||
receipt.write_text("\n".join(("schema=1", f"operation_id={OLD_ID}", f"status={status}",
|
||||
"phase=apply", f"checkpoint={'complete' if status == 'succeeded' else 'preflight'}",
|
||||
f"reason={'none' if status in {'succeeded', 'running', 'queued'} else 'command_failed'}",
|
||||
"created_at=100", "started_at=101", "updated_at=103", "finished_at=103", "")))
|
||||
receipt.chmod(0o600)
|
||||
tools = tmp_path / "tools"
|
||||
tools.mkdir()
|
||||
# Root UID is simulated; mode and symlink checks exercise the actual spool.
|
||||
stat = tools / "stat"
|
||||
stat.write_text("#!/usr/bin/env python3\nimport os,stat,sys\ns=os.lstat(sys.argv[-1]); print('0:'+format(stat.S_IMODE(s.st_mode),'o'))\n")
|
||||
sha = tools / "sha256sum"
|
||||
sha.write_text("#!/usr/bin/env python3\nimport hashlib,sys\nfor p in sys.argv[1:]:\n print(hashlib.sha256(open(p,'rb').read()).hexdigest()+' '+p)\n")
|
||||
systemctl = tools / "systemctl"
|
||||
if unit is None:
|
||||
unit = {"LoadState": "loaded", "ActiveState": "inactive", "SubState": "dead",
|
||||
"Result": "success", "ExecMainCode": "1", "ExecMainStatus": "0",
|
||||
"MainPID": "0", "Job": ""}
|
||||
unit_path = tmp_path / "unit.json"
|
||||
unit_path.write_text(json.dumps(unit))
|
||||
control = f'''#!/usr/bin/env python3
|
||||
import json,pathlib,sys
|
||||
assert sys.argv[1]=='show'
|
||||
log=pathlib.Path({str(tmp_path / 'calls')!r})
|
||||
count=len(log.read_text().splitlines()) if log.exists() else 0
|
||||
with log.open('a') as f: f.write('show\\n')
|
||||
unit=json.loads(pathlib.Path({str(unit_path)!r}).read_text())
|
||||
if count and {drift!r}=='unit': unit['Job']='45/run'
|
||||
if count and {drift!r}=='artifact': pathlib.Path({str(operation / 'apply')!r}).write_text('changed')
|
||||
if count and {drift!r}=='mode': pathlib.Path({str(operation / 'runner')!r}).chmod(0o755)
|
||||
if not count and {drift!r}=='status':
|
||||
p=pathlib.Path({str(operation / 'status')!r}); p.write_text(p.read_text().replace('status=succeeded','status=running'))
|
||||
if not count and {drift!r}=='manifest':
|
||||
p=pathlib.Path({str(manifest)!r}); p.write_text(p.read_text().replace({plan().digest!r},'0'*64))
|
||||
for k,v in unit.items(): print(k+'='+v)
|
||||
'''
|
||||
systemctl.write_text(control)
|
||||
for path in (stat, sha, systemctl): path.chmod(0o700)
|
||||
script = _fabric_operation_terminal_proof_script(OLD_ID, plan().digest)
|
||||
script = script.replace("/run/virt-cloud", str(base))
|
||||
script = script.replace("PATH=/usr/sbin:/usr/bin:/sbin:/bin", f"PATH={shlex.quote(str(tools))}:" + os.environ["PATH"])
|
||||
return script, operation, base, root
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shell", ["/bin/sh", "/bin/bash"])
|
||||
@pytest.mark.parametrize("variant", ["success", "failed", "collected_success", "collected_failed"])
|
||||
def test_real_terminal_proof_accepts_only_stopped_same_plan_spool(tmp_path, shell, variant):
|
||||
failed = variant.endswith("failed")
|
||||
unit = {"LoadState": "not-found" if variant.startswith("collected") else "loaded",
|
||||
"ActiveState": "inactive" if variant.startswith("collected") or not failed else "failed",
|
||||
"SubState": "dead" if variant.startswith("collected") or not failed else "failed",
|
||||
"Result": "success" if variant.startswith("collected") or not failed else "exit-code",
|
||||
"ExecMainCode": "0" if variant.startswith("collected") else "1",
|
||||
"ExecMainStatus": "1" if failed and not variant.startswith("collected") else "0",
|
||||
"MainPID": "0", "Job": ""}
|
||||
script, *_ = _proof_shell_fixture(tmp_path, status="failed" if failed else "succeeded", unit=unit)
|
||||
result = subprocess.run([shell], input=script, text=True, capture_output=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
adapter = object.__new__(FabricHostAdapter)
|
||||
adapter._run = lambda *_args, **_kwargs: result.stdout
|
||||
assert adapter.terminal_operation_proof(OLD_ID, plan().digest)["status"] == ("failed" if failed else "succeeded")
|
||||
assert (tmp_path / "calls").read_text().splitlines() == ["show", "show"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", [
|
||||
"running", "queued", "uncertain", "interrupted", "active_unit", "job", "pid", "unknown_load",
|
||||
"wrong_result", "wrong_exit", "unit_drift", "artifact_drift", "mode_drift", "status_drift",
|
||||
"manifest_drift", "plan", "hash",
|
||||
"parent_symlink", "parent_mode", "root_symlink", "root_mode", "extra_file", "artifact_symlink",
|
||||
"status_symlink", "manifest_mode", "status_mode", "operation_mode", "missing_manifest",
|
||||
])
|
||||
def test_real_terminal_proof_rejects_live_unknown_or_drifting_evidence(tmp_path, mutation):
|
||||
unit = {"LoadState": "loaded", "ActiveState": "inactive", "SubState": "dead",
|
||||
"Result": "success", "ExecMainCode": "1", "ExecMainStatus": "0", "MainPID": "0", "Job": ""}
|
||||
if mutation == "active_unit": unit.update(ActiveState="active", SubState="running")
|
||||
elif mutation == "job": unit["Job"] = "42/start"
|
||||
elif mutation == "pid": unit["MainPID"] = "123"
|
||||
elif mutation == "unknown_load": unit["LoadState"] = "error"
|
||||
elif mutation == "wrong_result": unit["Result"] = "exit-code"
|
||||
elif mutation == "wrong_exit": unit["ExecMainCode"] = "0"
|
||||
script, operation, base, root = _proof_shell_fixture(
|
||||
tmp_path, status=mutation if mutation in {"running", "queued", "uncertain", "interrupted"} else "succeeded",
|
||||
unit=unit, drift=mutation[:-6] if mutation.endswith("_drift") else None)
|
||||
if mutation == "plan":
|
||||
manifest = operation / "manifest"
|
||||
manifest.write_text(manifest.read_text().replace(plan().digest, "0" * 64))
|
||||
elif mutation == "hash": (operation / "apply").write_text("changed")
|
||||
elif mutation == "extra_file": (operation / ".hidden").write_text("extra")
|
||||
elif mutation in {"artifact_symlink", "status_symlink"}:
|
||||
path = operation / ("apply" if mutation == "artifact_symlink" else "status")
|
||||
target = tmp_path / "target"; target.write_bytes(path.read_bytes()); path.unlink(); path.symlink_to(target)
|
||||
elif mutation.endswith("_mode"):
|
||||
path = {"parent_mode": base, "root_mode": root, "manifest_mode": operation / "manifest",
|
||||
"status_mode": operation / "status", "operation_mode": operation}[mutation]
|
||||
path.chmod(0o755)
|
||||
elif mutation in {"parent_symlink", "root_symlink"}:
|
||||
path = base if mutation == "parent_symlink" else root
|
||||
target = path.with_name(path.name + "-real"); path.rename(target); path.symlink_to(target)
|
||||
elif mutation == "missing_manifest": (operation / "manifest").unlink()
|
||||
result = subprocess.run(["/bin/sh"], input=script, text=True, capture_output=True)
|
||||
assert result.returncode != 0, result.stdout
|
||||
@ -1,2 +1,2 @@
|
||||
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
|
||||
{assign var=venix_asset_version value='2026101403' scope='global'}
|
||||
{assign var=venix_asset_version value='2026101404' scope='global'}
|
||||
|
||||
Loading…
Reference in New Issue