From 0c8cf9d85329f71d613a75b1dafec326b2af0ed5 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 14 Oct 2026 17:18:53 +0800 Subject: [PATCH] fix: recover stopped Fabric operations before new reconciliation --- docs/FABRIC_HOST_ACCEPTANCE.md | 16 + docs/VALIDATION.md | 45 ++ docs/知识库/Memory/CURRENT.md | 20 + engine/tests/test_fabric_admission_receipt.py | 297 ++++++++++ .../tests/test_fabric_retained_operation.py | 451 ++++++++++++++ engine/virsh_panel/coordination.py | 58 ++ engine/virsh_panel/fabric.py | 560 ++++++++++++++---- engine/virsh_panel/internal.py | 42 +- web/app/Middleware/CloudApi.php | 13 +- web/app/Services/Cloud/CloudEngineClient.php | 25 + web/public/cloud/admin-fabric.js | 39 +- web/resources/views/venix/layout_assets.tpl | 2 +- web/tests/AdminFabricWizardBehaviorTest.mjs | 55 ++ .../CloudFabricAdmissionErrorContractTest.php | 318 ++++++++++ 14 files changed, 1803 insertions(+), 138 deletions(-) create mode 100644 engine/tests/test_fabric_admission_receipt.py create mode 100644 engine/tests/test_fabric_retained_operation.py create mode 100644 web/tests/CloudFabricAdmissionErrorContractTest.php diff --git a/docs/FABRIC_HOST_ACCEPTANCE.md b/docs/FABRIC_HOST_ACCEPTANCE.md index 1b31319e..6a6a89b7 100644 --- a/docs/FABRIC_HOST_ACCEPTANCE.md +++ b/docs/FABRIC_HOST_ACCEPTANCE.md @@ -3,6 +3,22 @@ 这份矩阵用于真实宿主机验收。仓库测试只证明生成脚本、能力解析、所有权回读和事务 契约;没有 AlmaLinux 10 或 Debian 13 真机证据时,不得把下面项目记为已通过。 +## 原 operation 未决占用与显式恢复 + +HTTP 502、断网或旧 operation 的诊断时间变化均不能证明本次写请求已接纳。 +先通过只读列表、Engine 协调占用与每台宿主的原 operation 回执核对;不要连续 +提交、直接改成功状态或删除协调文档。明确的接纳冲突应返回 409 和固定安全原因, +不应变成等待一个尚未产生的新 operation;真正结果未知仍须保持写入关闭。 + +显式“重新核对”恢复原 Fabric 占用时,必须逐台绑定原 ID、plan digest、受管 +spool 的文件归属与内容摘要,并证明原 systemd runner 已停止、无 PID/排队 job。 +`--collect` 回收后的 not-found 单独不构成证明,必须同时有完整、稳定的终态 +spool;任一任务仍运行、状态缺失、摘要变化或出现其他资源占用都保留原占用。 +恢复记录应与保护占用交接原子持久化,再接纳新一轮修复;普通 GET 和周期观察 +不得自动执行这类接管。原任务结束不代表 Fabric、实例网卡或租户流量已恢复。 +交接时发生真正的存储错误或 CAS 回包丢失,必须保留中间保护占用并人工核实; +不得把这一类 resource claim 当作普通旧 Fabric background claim 自动释放。 + ## 公网优先与后续内网 underlay - 当前公网组网可把 `fabric_endpoint_ip` 留空,但 SSH `host` 必须是本机持有的字面 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index ded00072..ae3c95a0 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,50 @@ # 本版验证记录 +## 2026-10-14:Fabric 旧未决占用阻断显式核对 + +- 现场已部署 `d50e6d5`,Engine 于 `08:47:49 UTC` 重启。`08:48:06 UTC` + 地区 reconcile 返回 HTTP 502;后续列表 GET 为 200,但 operation 仍是 + `791e7fd68d0d41fca5ec91e7d6ce053d`,创建与终态更新时间均停在 `08:04 UTC`。 + 本轮没有据此重放 POST,也没有修改运行数据库。 +- 只读确认原 Fabric plan/digest 可解析且资源目录约束通过。协调文档保留原 + `fabric_operation` 的 uncertain background claim,资源为 `catalog:network` + 与当前 Fabric;它阻断显式核对和周期回读。Engine 每 30 秒的通用异常日志未 + 投影该协调拒绝,PHP 又把未列入安全映射的内部错误泛化为 502。 +- 两台宿主原同名 transient unit 已被 collect 回收,完整 show 为 + `not-found/inactive/dead`、无排队 job;Alma 保留的原宿主 status 为 + `succeeded/apply/complete/none`。helper 第七行均仍为 v2,说明 v3 尚未通过新任务发布。 + 原任务执行结束与完整配置回读成功是两种证据;本轮不把 unit 成功当作私网恢复。 +- 同时发现七个无 operation、无资源的历史 request uncertain claim;它们不占 + 网络目录,但会影响维护接纳。未按时间、PID 或无资源条件批量删除或释放。 +- 修复仅允许显式核对处理与原 operation、Fabric 范围和后台类型完全一致的 + uncertain claim;全部节点须有合法派发记录,并取得同 ID/plan 的完整受管 spool + 与已停止单元证明。证明不依赖网络管理器,最多 16 台并发;不在接纳前重复完整 + 能力探测和网络回读,新后台任务仍执行这些门禁。 +- 终态证明验证父目录、五个文件的归属/权限/摘要、manifest/状态 schema 与两次 + 相同 systemd 停止读数,无 PID、无排队 job。collect 的 not-found 仅在完整终态 + spool 同时通过时接受;运行中、缺记录、跨计划、文件或单元漂移均不解除占用。 +- 原占用以同次 CAS 记录 resolution 并换成同范围保护 claim;锁内保存原失败回执 + 和诊断,再接纳新的修复 operation,最后释放保护。取消在本地交接完成后传播, + 不在交接中暴露无占用窗口;真实 CAS 回包丢失或持久化失败仍保留保护占用, + 须人工核对该中间状态,不自动接管任意 resource claim。 +- Engine 只对精确类型的 Fabric 接纳拒绝、未开始远端写入且无子 owner 的请求 + 返回安全 409 并释放本次空 request;其他未知错误保持原保护。PHP 与前端仅在 + 精确路由、固定错误码和 bare 409 合同内显示“本次未接纳”;5xx/断网仍只读 + 恢复,同一旧 uncertain 操作显示原 ID,不冒充新一轮已接纳。资产版本 `2026101404`。 +- 验证:相关十个既有 Python 模块 **466 passed**,新增原占用恢复/真实 Shell + 证明/CAS/取消 **69 passed**、真实 ASGI 接纳回执 **38 passed**,不重复合计 + **573 passed**;后续维护门禁 13 项复跑通过,不另计数。5 个 Python 文件 AST、 + 新证明脚本在 sh/dash/bash 下的语法、独立复核和 `git diff --check` 通过。 + 既有 Starlette/httpx 和 crypt 弃用警告保留。 +- PHP 8.5 WASM 新错误合同 **137 assertions**,既有 AdminFabricContract、 + CloudVpcEngineErrorContract、AdminFabricOperationProjection 三合同通过; + 完整 AdminFabricWizardBehavior 与 Node 语法检查通过。WASM/隔离 mock 验证 + 不等于 MariaDB、PHP-FPM、浏览器或真实宿主恢复验收。 +- 隔离 `d50e6d5` 的四个恢复正例均在原 claim 冲突处失败;仅为新测试 collection + 补了未在四例调用的证明函数符号,未替换旧接纳逻辑或执行旧版不存在的证明脚本。 + 旧 ASGI 的三个 uncertain 接纳拒绝复现 503;旧 PHP 十二个正向合同与新 JS 固定 + 文案用例亦先失败,再由当前修复通过。本轮尚未部署该交接修复或执行新的网络核对。 + ## 2026-10-14:Fabric tenant 集合写入与观察合同对齐 - `8172a3f` 已在现场部署并重启 Engine;新核对 diff --git a/docs/知识库/Memory/CURRENT.md b/docs/知识库/Memory/CURRENT.md index 20d25146..fd5309c2 100644 --- a/docs/知识库/Memory/CURRENT.md +++ b/docs/知识库/Memory/CURRENT.md @@ -10,6 +10,26 @@ tags: # 当前项目交接:Virt Cloud +## 2026-10-14 Fabric 更新后 502 与原 operation 占用 + +- **现场**:`d50e6d5` 已部署,Engine 重启后原 `791e…` 的终态时间仍为 + `08:04 UTC`。原 Fabric background uncertain 占用网络目录,显式核对与周期 + 回读均在接纳阶段被拒;内部错误被 PHP 泛化成 502,页面继续等待本轮回执。 +- **宿主证据**:两台 helper 仍 v2;原 transient unit 已 collect 回收、无排队 job, + Alma 原宿主 status 为 succeeded/complete。结束执行不代表精确配置回读成功。 +- **修复边界**:只允许显式核对按原 ID、计划摘要、稳定且归属完整的 spool 与停止 + unit 证明交接该 Fabric 的旧占用,保留审计记录,随后新任务重新探测并修复。 + 其他未知占用、未派发节点或证据失败均保留;周期观察不自动解除占用。 + 明确未接纳与真实结果未知使用不同的 HTTP/前端处理。 +- **部署边界**:本轮仍为源码修复与本地验证,未改运行数据库或重放网络任务。 + 七个历史无资源 request uncertain 占用未批量清理,不能因时间旧就释放。 +- **验证**:12 个不重叠 Python 模块 573 passed,含真实 Shell、ASGI、CAS 与取消 + 回归;PHP 新合同 137 assertions、三既有合同及完整 JS 行为通过,独立复核通过。 + 真正存储失败的中间保护占用仍须人工核对,不能扩大成任意占用自动清理。 +- 证据见 `docs/VALIDATION.md`、`docs/FABRIC_HOST_ACCEPTANCE.md`,沿用 + [[Concepts/Virt Cloud WireGuard Fabric与VPC L3私网]] 与 + [[Concepts/Virt Cloud VPC永久邻居与重复协调]]。 + ## 2026-10-14 Fabric tenant 集合回读不一致 - **现场新事实**:`8172a3f` 已部署,宿主 helper v2 已发布。新操作中 Debian diff --git a/engine/tests/test_fabric_admission_receipt.py b/engine/tests/test_fabric_admission_receipt.py new file mode 100644 index 00000000..90ab5ad2 --- /dev/null +++ b/engine/tests/test_fabric_admission_receipt.py @@ -0,0 +1,297 @@ +"""Real ASGI Fabric rejection receipts must not accumulate empty claims.""" + +import asyncio + +import pytest + +from test_engine_coordination import local_backends, request, running_engine # noqa: F401 +from test_fabric import plan_payload +from virsh_panel.coordination import ( + CoordinationClaimRejected, CoordinationError, DurableCoordinator, + current_execution, mark_execution_unknown, mark_remote_write_started, +) + + +OLD_OPERATION = "a" * 32 +TARGETS = [ + ("apply", "hk-fabric"), + ("reconcile", "hk-fabric"), + ("reconcile", "hk.fabric-1"), +] + + +def target(action, fabric_id): + if action == "apply": + value = plan_payload() + value["fabric_id"] = fabric_id + return "/internal/v1/fabric/apply", value + return f"/internal/v1/fabric/{fabric_id}/reconcile", {} + + +def old_owner(store, fabric_id, state="uncertain"): + owner = DurableCoordinator(store) + claim = owner.claim( + operation_id=OLD_OPERATION, + resources=("catalog:network", f"fabric:{fabric_id}"), kind="background", + metadata={"job_kind": "fabric_operation", "job_id": OLD_OPERATION}) + if state == "uncertain": + assert owner.retain(claim) + return claim, owner.snapshot() + + +@pytest.mark.parametrize("action,fabric_id", TARGETS) +@pytest.mark.parametrize("blocking_state", ["active", "uncertain"]) +def test_real_catalog_rejection_returns_409_without_new_request_claim( + tmp_path, monkeypatch, action, fabric_id, blocking_state): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, services): + original, before = old_owner(store, fabric_id, blocking_state) + remote_calls = [] + + async def forbidden_components(*_args, **_kwargs): + remote_calls.append(True) + pytest.fail("rejected Fabric admission reached a host") + + monkeypatch.setattr(engine.fabric, "_components", forbidden_components) + path, value = target(action, fabric_id) + code = ("coordination_uncertain" if blocking_state == "uncertain" + else "coordination_busy") + # Deliberately omit actor.operation_id, matching browser Fabric + # actions. Each attempt used to leave a new resources=[] claim. + for _attempt in range(3): + status, body = await request(engine, "POST", path, value) + assert status == 409, body + assert body == {"error": str(CoordinationError(code)), "error_code": code} + assert engine.coordinator.snapshot() == before + assert engine.coordinator.operation_claim(OLD_OPERATION) == before[original.token] + assert store.list("fabric_desired") == {} + assert store.list("fabric_node") == {} + assert store.list("operation") == {} + assert remote_calls == [] + assert services == {} + assert engine.fabric._operation_tasks == set() + + asyncio.run(exercise()) + + +@pytest.mark.parametrize("blocking_state", ["active", "uncertain"]) +def test_initial_request_claim_rejection_is_definitive_without_an_execution_context( + tmp_path, monkeypatch, blocking_state): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + blocker = DurableCoordinator(store) + original = blocker.claim(maintenance=True, kind="resource") + if blocking_state == "uncertain": + assert blocker.retain(original) + before = blocker.snapshot() + calls = [] + + async def forbidden(*_args, **_kwargs): + calls.append(True) + pytest.fail("globally rejected request reached the Fabric handler") + + monkeypatch.setattr(engine.fabric, "enqueue_reconcile", forbidden) + status, body = await request( + engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {}) + code = ("coordination_uncertain" if blocking_state == "uncertain" + else "coordination_busy") + assert status == 409, body + assert body == {"error": str(CoordinationError(code)), "error_code": code} + assert blocker.snapshot() == before + assert calls == [] + + asyncio.run(exercise()) + + +@pytest.mark.parametrize("code", ["coordination_busy", "coordination_uncertain"]) +def test_initial_generic_claim_error_is_not_a_definitive_rejection( + tmp_path, monkeypatch, code): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + _original, before = old_owner(store, "hk.fabric-1") + claim = engine.coordinator.claim + attempts = [] + + def unproven_claim(*args, **kwargs): + if kwargs.get("kind") == "request": + attempts.append(True) + raise CoordinationError(code, 503) + return claim(*args, **kwargs) + + monkeypatch.setattr(engine.coordinator, "claim", unproven_claim) + status, body = await request( + engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {}) + assert status == 503, body + assert body["error_code"] == code + assert attempts == [True] + assert engine.coordinator.snapshot() == before + + asyncio.run(exercise()) + + +class LookalikeClaimRejected(CoordinationClaimRejected): + """A subclass must not inherit the exact rejection receipt authority.""" + + +@pytest.mark.parametrize("action,fabric_id", TARGETS) +@pytest.mark.parametrize("failure", [ + "generic_busy", "generic_uncertain", "remote_write", "execution_unknown", + "background_child", "subclass", "unreviewed_code", +]) +def test_unproven_fabric_failure_remains_503_and_keeps_request_ownership( + tmp_path, monkeypatch, action, fabric_id, failure): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + original, before = old_owner(store, fabric_id) + seen = {} + + async def fail(*_args, **_kwargs): + context = current_execution() + seen["request"] = context.claim + assert context.claim.resources == () + assert context.claim.operation_id is None + if failure == "remote_write": + mark_remote_write_started() + elif failure == "execution_unknown": + mark_execution_unknown() + elif failure == "background_child": + child = context.coordinator.claim( + operation_id="b" * 32, resources=("fabric:other",), + execution_id=context.claim.execution_id, kind="background", + metadata={"job_kind": "fabric_operation", "job_id": "b" * 32}) + seen["child"] = context.coordinator.snapshot()[child.token] + if failure in {"generic_busy", "generic_uncertain"}: + code = ("coordination_busy" if failure == "generic_busy" + else "coordination_uncertain") + raise CoordinationError(code, 503) + if failure == "subclass": + raise LookalikeClaimRejected("coordination_uncertain", 503) + if failure == "unreviewed_code": + raise CoordinationClaimRejected("coordination_unavailable", 503) + raise CoordinationClaimRejected("coordination_uncertain", 503) + + monkeypatch.setattr(engine.fabric, "enqueue_" + action, fail) + path, value = target(action, fabric_id) + status, body = await request(engine, "POST", path, value) + expected_code = ("coordination_busy" if failure == "generic_busy" + else "coordination_unavailable" if failure == "unreviewed_code" + else "coordination_uncertain") + assert status == 503, body + assert body["error_code"] == expected_code + claims = engine.coordinator.snapshot() + assert claims[original.token] == before[original.token] + rejected = claims[seen["request"].token] + assert rejected["state"] == "uncertain" + assert rejected["resources"] == [] + assert rejected["operation_id"] is None + expected_tokens = {original.token, seen["request"].token} + if failure == "background_child": + child = seen["child"] + expected_tokens.add(child["token"]) + assert claims[child["token"]] == child + assert set(claims) == expected_tokens + assert store.list("fabric_desired") == {} + assert store.list("operation") == {} + + asyncio.run(exercise()) + + +def test_claim_rejection_on_fabric_probe_does_not_get_mutation_receipt( + tmp_path, monkeypatch): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + original, before = old_owner(store, "hk-fabric") + seen = {} + + async def fail(*_args, **_kwargs): + seen["request"] = current_execution().claim + raise CoordinationClaimRejected("coordination_uncertain", 503) + + monkeypatch.setattr(engine.fabric, "probe_hosts", fail) + status, body = await request( + engine, "POST", "/internal/v1/fabric/probe", {"host_ids": []}) + assert status == 503, body + assert body["error_code"] == "coordination_uncertain" + claims = engine.coordinator.snapshot() + assert claims[original.token] == before[original.token] + assert set(claims) == {original.token, seen["request"].token} + assert claims[seen["request"].token]["state"] == "uncertain" + + asyncio.run(exercise()) + + +@pytest.mark.parametrize("fault", [ + "snapshot_unavailable", "release_unavailable", "release_refused", "release_reply_lost", +]) +def test_rejection_receipt_requires_confirmed_snapshot_and_request_release( + tmp_path, monkeypatch, fault): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + original, before = old_owner(store, "hk.fabric-1") + seen = {"injected": False, "rejected": False} + snapshot = engine.coordinator.snapshot + release = engine.coordinator.release + + async def fail(*_args, **_kwargs): + seen["request"] = current_execution().claim + seen["rejected"] = True + raise CoordinationClaimRejected("coordination_uncertain", 503) + + def unavailable_snapshot(): + if fault == "snapshot_unavailable" and seen["rejected"] and not seen["injected"]: + seen["injected"] = True + raise CoordinationError("coordination_unavailable", 503) + return snapshot() + + def uncertain_release(claim): + if (fault != "snapshot_unavailable" and seen["rejected"] + and claim.token == seen["request"].token and not seen["injected"]): + seen["injected"] = True + if fault == "release_refused": + # Another owner retained this exact claim before release. + assert engine.coordinator.retain(claim) + return False + if fault == "release_reply_lost": + assert release(claim) + raise CoordinationError("coordination_unavailable", 503) + return release(claim) + + monkeypatch.setattr(engine.fabric, "enqueue_reconcile", fail) + monkeypatch.setattr(engine.coordinator, "snapshot", unavailable_snapshot) + monkeypatch.setattr(engine.coordinator, "release", uncertain_release) + status, body = await request( + engine, "POST", "/internal/v1/fabric/hk.fabric-1/reconcile", {}) + assert seen["injected"] is True + assert status == 503, body + assert body["error_code"] == ( + "coordination_uncertain" if fault == "release_refused" else "coordination_unavailable") + claims = snapshot() + assert claims[original.token] == before[original.token] + # A lost release reply may have removed this empty, unstarted + # request. It must never remove or alter the original resource fence. + if fault == "release_reply_lost": + assert claims == before + else: + assert set(claims) == {original.token, seen["request"].token} + if fault == "release_refused": + assert claims[seen["request"].token]["state"] == "uncertain" + assert store.list("fabric_desired") == {} + assert store.list("operation") == {} + + asyncio.run(exercise()) + + +@pytest.mark.parametrize("method,path", [ + ("PUT", "/internal/v1/fabric/apply"), + ("POST", "/internal/v1/fabric/hk.fabric-1/reconcile/extra"), +]) +def test_similar_method_or_route_does_not_expand_fabric_admission_receipt( + tmp_path, method, path): + async def exercise(): + async with running_engine(tmp_path) as (engine, store, _services): + _original, before = old_owner(store, "hk-fabric") + status, _body = await request(engine, method, path, {}) + assert status == 404 + assert engine.coordinator.snapshot() == before + + asyncio.run(exercise()) diff --git a/engine/tests/test_fabric_retained_operation.py b/engine/tests/test_fabric_retained_operation.py new file mode 100644 index 00000000..11149a91 --- /dev/null +++ b/engine/tests/test_fabric_retained_operation.py @@ -0,0 +1,451 @@ +"""Explicit recovery proves the exact old host jobs stopped before claim handoff.""" + +import asyncio +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import threading + +import pytest + +from virsh_panel.coordination import ( + CoordinationClaimRejected, CoordinationError, DurableCoordinator, + current_execution, execution_context, +) +from virsh_panel.fabric import ( + FabricCoordinator, FabricError, FabricHostAdapter, + _fabric_operation_terminal_proof_script, +) +from virsh_panel.internal import _DurableAsyncLock +from test_durable_fabric_jobs import FabricStore +from test_fabric import ( + AsyncCoordinatorAdapter, Registry, _active_fabric_record, _async_components, + _reconcile_observation, _remote_operation_status, plan, +) + +OLD_ID = "a" * 32 +SCOPE = ["catalog:network", "fabric:hk-fabric"] + + +class RecoveryAdapter(AsyncCoordinatorAdapter): + def terminal_operation_proof(self, operation_id, digest): + assert operation_id == OLD_ID and digest == plan().digest + self.manager.proof_calls += 1 + if self.manager.proof_error: + raise self.manager.proof_error + return _remote_operation_status(operation_id, self.manager.proof_status) + + def observe(self, current, host, keys): + result = super().observe(current, host, keys) + if self.manager.async_dispatch_calls == 0 and self.manager.old_readback_mismatch: + result.update(state="degraded", config_ok=False, transport_verified=False) + return result + + +def retained_fixture(monkeypatch, *, store=None): + current = plan() + store = store or FabricStore() + owner = DurableCoordinator(store) + catalog = _DurableAsyncLock(owner, ("catalog:network",)) + components = _async_components(current) + for component in components.values(): + component.manager.proof_calls = 0 + component.manager.proof_status = "succeeded" + component.manager.proof_error = None + component.manager.old_readback_mismatch = True + fabric = FabricCoordinator(store, Registry(components), catalog_lock=lambda: catalog) + desired = _active_fabric_record(current) + desired["operation"] = fabric._new_root_operation(OLD_ID, "reconcile", 100) + desired["operation"].update(status="uncertain", started_at=101, finished_at=103) + for node in desired["nodes"]: + node["operation"] = fabric._new_node_operation(OLD_ID, "apply", 100) + node["operation"].update(status="uncertain", started_at=101, finished_at=103) + fabric._invalidate(desired) + fabric._save("fabric_desired", current.fabric_id, desired) + previous_owner = DurableCoordinator(store) + old = previous_owner.claim( + operation_id=OLD_ID, resources=SCOPE, kind="background", + metadata={"job_kind": "fabric_operation", "job_id": OLD_ID}) + assert previous_owner.retain(old) + monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", RecoveryAdapter) + monkeypatch.setattr("virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) + return store, owner, fabric, components, old + + +@pytest.mark.parametrize("old_status", ["succeeded", "failed"]) +@pytest.mark.parametrize("mismatch", [False, True]) +def test_explicit_terminal_recovery_closes_old_receipt_and_transfers_new_job( + monkeypatch, old_status, mismatch): + async def run(): + store, owner, fabric, components, old = retained_fixture(monkeypatch) + for component in components.values(): + component.manager.proof_status = old_status + component.manager.old_readback_mismatch = mismatch + parent = owner.claim(kind="request") + try: + with execution_context(owner, parent): + result = await fabric.enqueue_reconcile("hk-fabric") + assert result["operation"]["id"] != OLD_ID + current = fabric._load("fabric_desired", "hk-fabric") + receipt = current["reconciled_operation"] + assert receipt["operation"]["id"] == OLD_ID + assert receipt["operation"]["status"] == "failed" + assert all(node["remote_status"] == old_status for node in receipt["nodes"]) + assert all(node["operation"]["status"] == receipt["operation"]["status"] + for node in receipt["nodes"]) + assert receipt["claim_token"] == old.token + row = json.loads(store.get("coordination", "admission-v1")) + assert len(row["resolutions"]) == 1 + assert row["resolutions"][0]["operation_id"] == OLD_ID + assert old.token not in row["claims"] + children = [claim for claim in row["claims"].values() if claim["kind"] == "background"] + assert len(children) == 1 + assert children[0]["operation_id"] == result["operation"]["id"] + with pytest.raises(CoordinationClaimRejected): + DurableCoordinator(store).claim(maintenance=True, kind="update") + assert owner.release(parent) + await asyncio.wait_for(asyncio.gather(*tuple(fabric._operation_tasks)), 2) + assert owner.snapshot() == {} + assert all(component.manager.proof_calls == 1 for component in components.values()) + assert all(component.manager.async_dispatch_calls == int(mismatch) + for component in components.values()) + finally: + await fabric.shutdown_operations() + fabric.close() + asyncio.run(run()) + + +@pytest.mark.parametrize("mutation", [ + "active", "other_operation", "extra_resource", "wrong_kind", "wrong_job", "maintenance", + "extra_claim", "node_id", "node_phase", "node_checkpoint", "node_pending", "root_digest", + "root_kind", "root_status", "root_times", "host_profile", + "running_proof", "queued_proof", "uncertain_proof", "proof_failure", +]) +def test_unproven_or_different_owner_is_never_resolved(monkeypatch, mutation): + async def run(): + store, owner, fabric, components, old = retained_fixture(monkeypatch) + doc = json.loads(store.get("coordination", "admission-v1")) + claim = doc["claims"][old.token] + if mutation == "active": claim["state"] = "active" + elif mutation == "other_operation": claim["operation_id"] = "b" * 32 + elif mutation == "extra_resource": claim["resources"].append("host:host-a") + elif mutation == "wrong_kind": claim["kind"] = "resource" + elif mutation == "wrong_job": claim["metadata"]["job_kind"] = "other_job" + elif mutation == "maintenance": claim["maintenance"] = True + elif mutation == "extra_claim": + extra = {**claim, "token": "d" * 64, "execution_id": "e" * 64} + doc["claims"][extra["token"]] = extra + store.set("coordination", "admission-v1", json.dumps(doc)) + desired = fabric._load("fabric_desired", "hk-fabric") + node = desired["nodes"][1]["operation"] + if mutation == "node_id": node["id"] = "b" * 32 + elif mutation == "node_phase": node["phase"] = "unknown" + elif mutation == "node_checkpoint": node["checkpoint"] = "unknown" + elif mutation == "node_pending": node.update(phase="identity", checkpoint="dispatch", status="pending") + elif mutation == "root_digest": desired["plan_digest"] = "0" * 64 + elif mutation == "root_kind": desired["operation"]["kind"] = "other" + elif mutation == "root_status": desired["operation"]["status"] = "other" + elif mutation == "root_times": desired["operation"]["created_at"] = True + elif mutation == "host_profile": components["host-b"].manager.profile["host_id"] = "host-c" + fabric._save("fabric_desired", "hk-fabric", desired) + manager = components["host-b"].manager + if mutation.endswith("_proof"): manager.proof_status = mutation[:-6] + elif mutation == "proof_failure": + manager.proof_error = FabricError("unsafe private stderr", 502, "fabric_operation_status_invalid") + # Maintenance admission itself rejects the new request before it exists. + if mutation == "maintenance": + with pytest.raises(CoordinationClaimRejected): owner.claim(kind="request") + assert owner.snapshot()[old.token]["state"] == "uncertain" + fabric.close() + return + parent = owner.claim(kind="request") + before = store.get("fabric_desired", "hk-fabric") + try: + with execution_context(owner, parent), pytest.raises((FabricError, CoordinationError)): + await fabric.enqueue_reconcile("hk-fabric") + assert store.get("fabric_desired", "hk-fabric") == before + assert old.token in owner.snapshot() + assert json.loads(store.get("coordination", "admission-v1"))["resolutions"] == [] + assert not fabric._operation_tasks + assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) + finally: + owner.release(parent) + fabric.close() + asyncio.run(run()) + + +def test_periodic_observation_keeps_retained_claim_and_does_not_probe_or_dispatch(monkeypatch): + async def run(): + store, owner, fabric, components, old = retained_fixture(monkeypatch) + before = store.get("fabric_desired", "hk-fabric") + await fabric.reconcile_all() + assert store.get("fabric_desired", "hk-fabric") == before + assert list(owner.snapshot()) == [old.token] + assert all(component.manager.proof_calls == 0 and component.manager.async_dispatch_calls == 0 + for component in components.values()) + fabric.close() + asyncio.run(run()) + + +@pytest.mark.parametrize("change", ["metadata", "updated_at", "resources"]) +def test_atomic_handoff_rejects_stale_claim_snapshot_without_resolution(change): + store = FabricStore() + old_owner, new_owner = DurableCoordinator(store), DurableCoordinator(store) + old = old_owner.claim(operation_id=OLD_ID, resources=SCOPE, kind="background") + old_owner.retain(old) + snapshot = new_owner.snapshot()[old.token] + if change == "metadata": old_owner.update_metadata(old, {"later": True}) + elif change == "updated_at": + doc = json.loads(store.get("coordination", "admission-v1")) + doc["claims"][old.token]["updated_at"] += 1 + store.set("coordination", "admission-v1", json.dumps(doc)) + else: + doc = json.loads(store.get("coordination", "admission-v1")) + doc["claims"][old.token]["resources"].append("host:host-a") + store.set("coordination", "admission-v1", json.dumps(doc)) + with pytest.raises(CoordinationClaimRejected): + new_owner.resolve_and_claim(snapshot, "terminal proof", execution_id="f" * 64) + doc = json.loads(store.get("coordination", "admission-v1")) + assert old.token in doc["claims"] and doc["resolutions"] == [] + + +def test_every_atomic_commit_preserves_fence_and_failed_handoff_stays_exclusive(monkeypatch): + class InspectStore(FabricStore): + inspect = False + def compare_and_set(self, kind, key, expected, desired): + if self.inspect and kind == "coordination": + claims = json.loads(desired)["claims"].values() + assert any(set(SCOPE) <= set(row["resources"]) for row in claims) + return super().compare_and_set(kind, key, expected, desired) + async def run(): + store = InspectStore() + store, owner, fabric, _components, old = retained_fixture(monkeypatch, store=store) + parent = owner.claim(kind="request") + store.inspect = True + original_save = fabric._save + def fail_terminal(kind, key, value): + if kind == "fabric_desired" and "reconciled_operation" in value: + raise RuntimeError("injected persistence failure") + return original_save(kind, key, value) + monkeypatch.setattr(fabric, "_save", fail_terminal) + try: + with execution_context(owner, parent), pytest.raises(RuntimeError): + await fabric.enqueue_reconcile("hk-fabric") + assert old.token not in owner.snapshot() + replacement = next(row for row in owner.snapshot().values() if row["resources"] == SCOPE) + assert replacement["state"] == "uncertain" and replacement["resources"] == SCOPE + with pytest.raises(CoordinationClaimRejected): + DurableCoordinator(store).claim(resources=("catalog:network",)) + assert fabric._load("fabric_desired", "hk-fabric")["operation"]["id"] == OLD_ID + finally: + owner.release(parent) + fabric.close() + asyncio.run(run()) + + +def test_lost_atomic_handoff_response_keeps_replacement_fence(monkeypatch): + class LostReplyStore(FabricStore): + lose_reply = False + + def compare_and_set(self, kind, key, expected, desired): + committed = super().compare_and_set(kind, key, expected, desired) + if committed and self.lose_reply and kind == "coordination" and json.loads(desired)["resolutions"]: + self.lose_reply = False + raise RuntimeError("CAS committed, reply lost") + return committed + + async def run(): + store, owner, fabric, _components, old = retained_fixture(monkeypatch, store=LostReplyStore()) + parent = owner.claim(kind="request") + store.lose_reply = True + try: + with execution_context(owner, parent), pytest.raises(CoordinationError) as raised: + await fabric.enqueue_reconcile("hk-fabric") + assert raised.value.error_code == "coordination_unavailable" + rows = owner.snapshot() + assert old.token not in rows + assert any(row["resources"] == SCOPE for row in rows.values()) + assert fabric._load("fabric_desired", "hk-fabric")["operation"]["id"] == OLD_ID + assert not fabric._operation_tasks + with pytest.raises(CoordinationClaimRejected): + DurableCoordinator(store).claim(maintenance=True, kind="update") + finally: + owner.release(parent) + fabric.close() + + asyncio.run(run()) + + +@pytest.mark.parametrize("phase", ["proof", "handoff", "new_owner"]) +def test_cancellation_cannot_release_fence_before_host_proof_or_new_owner(monkeypatch, phase): + async def run(): + store, owner, fabric, components, old = retained_fixture(monkeypatch) + entered, resume = threading.Event(), threading.Event() + original_proof = RecoveryAdapter.terminal_operation_proof + original_handoff = owner.resolve_and_claim + original_claim = owner.claim + def block_proof(adapter, operation_id, digest): + if phase == "proof" and adapter.manager.host_id == "host-a": + entered.set(); assert resume.wait(3) + return original_proof(adapter, operation_id, digest) + def block_handoff(*args, **kwargs): + result = original_handoff(*args, **kwargs) + if phase == "handoff": entered.set(); assert resume.wait(3) + return result + def block_child(**kwargs): + result = original_claim(**kwargs) + if phase == "new_owner" and kwargs.get("kind") == "background": + entered.set(); assert resume.wait(3) + return result + monkeypatch.setattr(RecoveryAdapter, "terminal_operation_proof", block_proof) + monkeypatch.setattr(owner, "resolve_and_claim", block_handoff) + monkeypatch.setattr(owner, "claim", block_child) + parent = owner.claim(kind="request") + with execution_context(owner, parent): task = asyncio.create_task(fabric.enqueue_reconcile("hk-fabric")) + try: + for _ in range(300): + if entered.is_set(): break + await asyncio.sleep(.002) + assert entered.is_set() + task.cancel() + await asyncio.sleep(.01) + with pytest.raises(CoordinationClaimRejected): + DurableCoordinator(store).claim(maintenance=True, kind="update") + resume.set() + with pytest.raises(asyncio.CancelledError): await task + if phase == "proof": + assert old.token in owner.snapshot() + assert not fabric._operation_tasks + else: + assert old.token not in owner.snapshot() + await asyncio.gather(*tuple(fabric._operation_tasks)) + assert not any(row["kind"] == "resource" for row in owner.snapshot().values()) + owner.release(parent) + finally: + resume.set() + await fabric.shutdown_operations() + fabric.close() + asyncio.run(run()) + + +def _proof_shell_fixture(tmp_path, *, status="succeeded", unit=None, drift=None): + base = tmp_path / "virt-cloud" + root = base / "fabric-operations" + operation = root / OLD_ID + operation.mkdir(parents=True) + for path in (base, root, operation): path.chmod(0o700) + digests = [] + for name in ("apply", "update", "runner"): + path = operation / name + path.write_text(f"#!/bin/sh\n# {name}\nexit 0\n") + path.chmod(0o700) + digests.append(hashlib.sha256(path.read_bytes()).hexdigest()) + manifest = operation / "manifest" + manifest.write_text("\n".join(("schema=1", f"operation_id={OLD_ID}", + f"plan_digest={plan().digest}", f"script_digest={digests[0]}", + f"update_digest={digests[1]}", f"runner_digest={digests[2]}", "created_at=100", ""))) + manifest.chmod(0o600) + receipt = operation / "status" + receipt.write_text("\n".join(("schema=1", f"operation_id={OLD_ID}", f"status={status}", + "phase=apply", f"checkpoint={'complete' if status == 'succeeded' else 'preflight'}", + f"reason={'none' if status in {'succeeded', 'running', 'queued'} else 'command_failed'}", + "created_at=100", "started_at=101", "updated_at=103", "finished_at=103", ""))) + receipt.chmod(0o600) + tools = tmp_path / "tools" + tools.mkdir() + # Root UID is simulated; mode and symlink checks exercise the actual spool. + stat = tools / "stat" + stat.write_text("#!/usr/bin/env python3\nimport os,stat,sys\ns=os.lstat(sys.argv[-1]); print('0:'+format(stat.S_IMODE(s.st_mode),'o'))\n") + sha = tools / "sha256sum" + sha.write_text("#!/usr/bin/env python3\nimport hashlib,sys\nfor p in sys.argv[1:]:\n print(hashlib.sha256(open(p,'rb').read()).hexdigest()+' '+p)\n") + systemctl = tools / "systemctl" + if unit is None: + unit = {"LoadState": "loaded", "ActiveState": "inactive", "SubState": "dead", + "Result": "success", "ExecMainCode": "1", "ExecMainStatus": "0", + "MainPID": "0", "Job": ""} + unit_path = tmp_path / "unit.json" + unit_path.write_text(json.dumps(unit)) + control = f'''#!/usr/bin/env python3 +import json,pathlib,sys +assert sys.argv[1]=='show' +log=pathlib.Path({str(tmp_path / 'calls')!r}) +count=len(log.read_text().splitlines()) if log.exists() else 0 +with log.open('a') as f: f.write('show\\n') +unit=json.loads(pathlib.Path({str(unit_path)!r}).read_text()) +if count and {drift!r}=='unit': unit['Job']='45/run' +if count and {drift!r}=='artifact': pathlib.Path({str(operation / 'apply')!r}).write_text('changed') +if count and {drift!r}=='mode': pathlib.Path({str(operation / 'runner')!r}).chmod(0o755) +if not count and {drift!r}=='status': + p=pathlib.Path({str(operation / 'status')!r}); p.write_text(p.read_text().replace('status=succeeded','status=running')) +if not count and {drift!r}=='manifest': + p=pathlib.Path({str(manifest)!r}); p.write_text(p.read_text().replace({plan().digest!r},'0'*64)) +for k,v in unit.items(): print(k+'='+v) +''' + systemctl.write_text(control) + for path in (stat, sha, systemctl): path.chmod(0o700) + script = _fabric_operation_terminal_proof_script(OLD_ID, plan().digest) + script = script.replace("/run/virt-cloud", str(base)) + script = script.replace("PATH=/usr/sbin:/usr/bin:/sbin:/bin", f"PATH={shlex.quote(str(tools))}:" + os.environ["PATH"]) + return script, operation, base, root + + +@pytest.mark.parametrize("shell", ["/bin/sh", "/bin/bash"]) +@pytest.mark.parametrize("variant", ["success", "failed", "collected_success", "collected_failed"]) +def test_real_terminal_proof_accepts_only_stopped_same_plan_spool(tmp_path, shell, variant): + failed = variant.endswith("failed") + unit = {"LoadState": "not-found" if variant.startswith("collected") else "loaded", + "ActiveState": "inactive" if variant.startswith("collected") or not failed else "failed", + "SubState": "dead" if variant.startswith("collected") or not failed else "failed", + "Result": "success" if variant.startswith("collected") or not failed else "exit-code", + "ExecMainCode": "0" if variant.startswith("collected") else "1", + "ExecMainStatus": "1" if failed and not variant.startswith("collected") else "0", + "MainPID": "0", "Job": ""} + script, *_ = _proof_shell_fixture(tmp_path, status="failed" if failed else "succeeded", unit=unit) + result = subprocess.run([shell], input=script, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + adapter = object.__new__(FabricHostAdapter) + adapter._run = lambda *_args, **_kwargs: result.stdout + assert adapter.terminal_operation_proof(OLD_ID, plan().digest)["status"] == ("failed" if failed else "succeeded") + assert (tmp_path / "calls").read_text().splitlines() == ["show", "show"] + + +@pytest.mark.parametrize("mutation", [ + "running", "queued", "uncertain", "interrupted", "active_unit", "job", "pid", "unknown_load", + "wrong_result", "wrong_exit", "unit_drift", "artifact_drift", "mode_drift", "status_drift", + "manifest_drift", "plan", "hash", + "parent_symlink", "parent_mode", "root_symlink", "root_mode", "extra_file", "artifact_symlink", + "status_symlink", "manifest_mode", "status_mode", "operation_mode", "missing_manifest", +]) +def test_real_terminal_proof_rejects_live_unknown_or_drifting_evidence(tmp_path, mutation): + unit = {"LoadState": "loaded", "ActiveState": "inactive", "SubState": "dead", + "Result": "success", "ExecMainCode": "1", "ExecMainStatus": "0", "MainPID": "0", "Job": ""} + if mutation == "active_unit": unit.update(ActiveState="active", SubState="running") + elif mutation == "job": unit["Job"] = "42/start" + elif mutation == "pid": unit["MainPID"] = "123" + elif mutation == "unknown_load": unit["LoadState"] = "error" + elif mutation == "wrong_result": unit["Result"] = "exit-code" + elif mutation == "wrong_exit": unit["ExecMainCode"] = "0" + script, operation, base, root = _proof_shell_fixture( + tmp_path, status=mutation if mutation in {"running", "queued", "uncertain", "interrupted"} else "succeeded", + unit=unit, drift=mutation[:-6] if mutation.endswith("_drift") else None) + if mutation == "plan": + manifest = operation / "manifest" + manifest.write_text(manifest.read_text().replace(plan().digest, "0" * 64)) + elif mutation == "hash": (operation / "apply").write_text("changed") + elif mutation == "extra_file": (operation / ".hidden").write_text("extra") + elif mutation in {"artifact_symlink", "status_symlink"}: + path = operation / ("apply" if mutation == "artifact_symlink" else "status") + target = tmp_path / "target"; target.write_bytes(path.read_bytes()); path.unlink(); path.symlink_to(target) + elif mutation.endswith("_mode"): + path = {"parent_mode": base, "root_mode": root, "manifest_mode": operation / "manifest", + "status_mode": operation / "status", "operation_mode": operation}[mutation] + path.chmod(0o755) + elif mutation in {"parent_symlink", "root_symlink"}: + path = base if mutation == "parent_symlink" else root + target = path.with_name(path.name + "-real"); path.rename(target); path.symlink_to(target) + elif mutation == "missing_manifest": (operation / "manifest").unlink() + result = subprocess.run(["/bin/sh"], input=script, text=True, capture_output=True) + assert result.returncode != 0, result.stdout diff --git a/engine/virsh_panel/coordination.py b/engine/virsh_panel/coordination.py index 8a953702..9f4b528a 100644 --- a/engine/virsh_panel/coordination.py +++ b/engine/virsh_panel/coordination.py @@ -390,6 +390,64 @@ class DurableCoordinator: return True, True return self._change(mutate) + def resolve_and_claim(self, claim: Mapping[str, Any], evidence: str, *, + execution_id: str) -> Claim: + """Explicitly replace one proven stopped owner without an admission gap. + + The supplied snapshot must still match in full. The replacement keeps + its exact resource fence while the caller persists the old operation's + terminal receipt and transfers ownership to a new background job. + """ + if not isinstance(evidence, str) or not evidence.strip() or len(evidence) > 1024: + raise ValueError("Explicit reconciliation evidence is required") + _identity(claim) + if (not isinstance(execution_id, str) or not _TOKEN.fullmatch(execution_id) + or execution_id in {claim.get("operation_id"), self.owner_id, + claim.get("execution_id")}): + raise ValueError("Reconciliation requires a new execution identity") + token = secrets.token_hex(32) + + def mutate(document): + current = self._owned(document, claim) + if (current is None or current != dict(claim) + or current["state"] != "uncertain"): + raise CoordinationClaimRejected("coordination_uncertain", 503) + for existing in document["claims"].values(): + if existing["token"] == current["token"]: + continue + if existing["maintenance"] or ( + (existing["state"] == "uncertain" + or existing["execution_id"] != execution_id) + and (existing["operation_id"] == current["operation_id"] + or bool(set(current["resources"]).intersection(existing["resources"]))) + ): + unknown = existing["state"] == "uncertain" + raise CoordinationClaimRejected( + "coordination_uncertain" if unknown else "coordination_busy", + 503 if unknown else 409) + now = int(time.time()) + replacement = { + **current, "token": token, "execution_id": execution_id, + "owner_id": self.owner_id, "state": "active", "kind": "resource", + "metadata": {"reconciled_operation_id": current["operation_id"]}, + "created_at": now, "updated_at": now, + } + replacement.pop("reason_code", None) + receipts = document.setdefault("resolutions", []) + receipts.append({ + "token": current["token"], "operation_id": current["operation_id"], + "resolved_by": self.owner_id, "resolved_at": now, + "evidence": evidence.strip(), "replacement_token": token, + }) + document["resolutions"] = receipts[-100:] + del document["claims"][current["token"]] + document["claims"][token] = replacement + return True, Claim(token, execution_id, self.owner_id, + replacement["operation_id"], tuple(replacement["resources"]), + "resource", dict(replacement["metadata"])) + + return self._change(mutate) + def configure_default_coordinator(coordinator: DurableCoordinator | None) -> None: global _default_coordinator diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 12390048..225bc626 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -18,11 +18,15 @@ import re import shlex import time import uuid +from contextlib import nullcontext from dataclasses import dataclass from typing import Any, Callable, Mapping from .remote_host import RemoteHostError from .durable_jobs import create_durable_task +from .coordination import ( + CoordinationClaimRejected, current_execution, execution_context, +) from .shell_contracts import ( BRIDGE_PORT_RECORDS_SH, IP_RULE_CONVERGENCE_SH, LOCAL_NEIGHBOR_SH, VPC_SYSCTL_VALIDATION_SH, WG_ALLOWED_IPS_SH, @@ -1534,6 +1538,102 @@ printf '%s\n' \ ''' +def _fabric_operation_terminal_proof_script(operation_id: str, plan_digest: str) -> str: + """Read immutable spool and stopped unit evidence; never start a runner.""" + operation_id = _validate_operation_id(operation_id) + if not isinstance(plan_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", plan_digest): + raise FabricError("Fabric 计划摘要无效", 500, "fabric_plan_corrupt") + # Reuse the status reader's ownership/schema checks and exact projection. + parent_proof = f''' +for parent in /run/virt-cloud {_q(FABRIC_OPERATION_ROOT)}; do + [ -d "$parent" ] && [ ! -L "$parent" ] || exit 71 + [ "$(stat -c '%u:%a' "$parent" 2>/dev/null)" = 0:700 ] || exit 71 +done +''' + status_reader = _fabric_operation_status_script(operation_id).replace( + 'STATUS="$OPERATION_DIR/status"\n', 'STATUS="$OPERATION_DIR/status"\n' + parent_proof) + initial_spool = ''' +for artifact in manifest status apply update runner; do + [ -f "$OPERATION_DIR/$artifact" ] && [ ! -L "$OPERATION_DIR/$artifact" ] || exit 71 + case "$artifact" in manifest|status) artifact_mode=600 ;; *) artifact_mode=700 ;; esac + [ "$(stat -c '%u:%a' "$OPERATION_DIR/$artifact" 2>/dev/null)" = "0:$artifact_mode" ] || exit 71 +done +spool_snapshot=$(sha256sum "$OPERATION_DIR/manifest" "$STATUS" "$OPERATION_DIR/apply" "$OPERATION_DIR/update" "$OPERATION_DIR/runner") +''' + status_reader = status_reader.replace( + '[ "$(wc -l < "$STATUS" | tr -d \' \')" = 10 ] || exit 71\n', + initial_spool + '[ "$(wc -l < "$STATUS" | tr -d \' \')" = 10 ] || exit 71\n') + return status_reader + f''' +PLAN_DIGEST={_q(plan_digest)} +UNIT={_q(f"{FABRIC_OPERATION_UNIT_PREFIX}{operation_id}.service")} +[ "$state" = succeeded ] || [ "$state" = failed ] || exit 71 +MANIFEST="$OPERATION_DIR/manifest" +[ -f "$MANIFEST" ] && [ ! -L "$MANIFEST" ] || exit 71 +[ "$(stat -c '%u:%a' "$MANIFEST" 2>/dev/null)" = 0:600 ] || exit 71 +[ "$(wc -l < "$MANIFEST" | tr -d ' ')" = 7 ] || exit 71 +[ "$(sed -n '1p' "$MANIFEST")" = schema=1 ] || exit 71 +[ "$(sed -n '2p' "$MANIFEST")" = "operation_id=$OPERATION_ID" ] || exit 71 +[ "$(sed -n '3p' "$MANIFEST")" = "plan_digest=$PLAN_DIGEST" ] || exit 71 +[ "$(sed -n '7p' "$MANIFEST")" = "created_at=$created_at" ] || exit 71 +validate_spool() {{ +[ -d "$OPERATION_DIR" ] && [ ! -L "$OPERATION_DIR" ] || return 1 +[ "$(stat -c '%u:%a' "$OPERATION_DIR" 2>/dev/null)" = 0:700 ] || return 1 +file_count=0 +for artifact in "$OPERATION_DIR"/* "$OPERATION_DIR"/.[!.]* "$OPERATION_DIR"/..?*; do + [ -e "$artifact" ] || [ -L "$artifact" ] || continue + case "${{artifact##*/}}" in manifest|status|apply|update|runner) : ;; *) return 1 ;; esac + file_count=$((file_count + 1)) +done +[ "$file_count" = 5 ] || return 1 +for artifact in manifest status; do + [ -f "$OPERATION_DIR/$artifact" ] && [ ! -L "$OPERATION_DIR/$artifact" ] || return 1 + [ "$(stat -c '%u:%a' "$OPERATION_DIR/$artifact" 2>/dev/null)" = 0:600 ] || return 1 +done +for artifact in apply update runner; do + [ -f "$OPERATION_DIR/$artifact" ] && [ ! -L "$OPERATION_DIR/$artifact" ] || return 1 + [ "$(stat -c '%u:%a' "$OPERATION_DIR/$artifact" 2>/dev/null)" = 0:700 ] || return 1 + case "$artifact" in apply) digest_line=4; digest_key=script_digest ;; update) digest_line=5; digest_key=update_digest ;; runner) digest_line=6; digest_key=runner_digest ;; esac + artifact_digest=$(sed -n "${{digest_line}}s/^$digest_key=//p" "$MANIFEST") + case "$artifact_digest" in *[!0-9a-f]*) return 1 ;; esac + [ "${{#artifact_digest}}" = 64 ] || return 1 + [ "$(sha256sum "$OPERATION_DIR/$artifact" | awk '{{print $1}}')" = "$artifact_digest" ] || return 1 +done +}} +validate_spool || exit 71 +read_stopped_unit() {{ + unit_snapshot=$(systemctl show "$UNIT" --no-pager \ + --property=LoadState --property=ActiveState --property=SubState \ + --property=Result --property=ExecMainCode --property=ExecMainStatus \ + --property=MainPID --property=Job) || return 1 + [ "$(printf '%s\\n' "$unit_snapshot" | wc -l | tr -d ' ')" = 8 ] || return 1 + for property in LoadState ActiveState SubState Result ExecMainCode ExecMainStatus MainPID Job; do + [ "$(printf '%s\\n' "$unit_snapshot" | grep -c "^$property=")" = 1 ] || return 1 + done + unit_load=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^LoadState=//p') + unit_active=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^ActiveState=//p') + unit_sub=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^SubState=//p') + unit_result=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^Result=//p') + unit_code=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^ExecMainCode=//p') + unit_status=$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^ExecMainStatus=//p') + [ "$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^MainPID=//p')" = 0 ] || return 1 + [ -z "$(printf '%s\\n' "$unit_snapshot" | sed -n 's/^Job=//p')" ] || return 1 + case "$unit_load:$unit_active:$unit_sub:$unit_result:$unit_code:$unit_status:$state" in + loaded:inactive:dead:success:1:0:succeeded|loaded:failed:failed:exit-code:1:1:failed) : ;; + not-found:inactive:dead:success:0:0:succeeded|not-found:inactive:dead:success:0:0:failed) : ;; + *) return 1 ;; + esac +}} +read_stopped_unit || exit 71 +first_unit_snapshot=$unit_snapshot +read_stopped_unit || exit 71 +[ "$first_unit_snapshot" = "$unit_snapshot" ] || exit 71 +{parent_proof} +validate_spool || exit 71 +[ "$spool_snapshot" = "$(sha256sum "$MANIFEST" "$STATUS" "$OPERATION_DIR/apply" "$OPERATION_DIR/update" "$OPERATION_DIR/runner")" ] || exit 71 +printf '%s\\n' "__VC_FABRIC_OPERATION_TERMINAL_PROOF__=$PLAN_DIGEST" +''' + + def _fabric_operation_dispatch_script( operation_id: str, rendered_apply: str, plan_digest: str) -> str: operation_id = _validate_operation_id(operation_id) @@ -8826,6 +8926,22 @@ class FabricHostAdapter: "fabric_operation_not_found", "observe") return _parse_operation_status(output, safe_operation_id) + def terminal_operation_proof(self, operation_id: str, plan_digest: str) -> dict[str, Any]: + output = self._run( + _fabric_operation_terminal_proof_script(operation_id, plan_digest), + failure_phase="observe", + timeout_seconds=FABRIC_REMOTE_READ_TIMEOUT_SECONDS, + ) + lines = output.splitlines() + if not lines or lines[-1] != f"__VC_FABRIC_OPERATION_TERMINAL_PROOF__={plan_digest}": + raise FabricError("宿主机 Fabric 任务终态证明无效", 409, + "fabric_operation_status_invalid", "observe") + status = _parse_operation_status("\n".join(lines[:-1]), operation_id) + if status["status"] not in {"succeeded", "failed"}: + raise FabricError("宿主机 Fabric 任务仍未确认结束", 409, + "fabric_operation_status_invalid", "observe") + return status + def observe(self, plan: FabricPlan, host: FabricNode, public_keys: Mapping[str, str]) -> dict[str, Any]: targets = tuple( @@ -10432,6 +10548,195 @@ class FabricCoordinator: return self._public(desired) async def enqueue_reconcile(self, fabric_id: str) -> dict[str, Any]: + """Explicitly reconcile a stopped retained operation before new work.""" + if not isinstance(fabric_id, str) or not _FABRIC_KEY.fullmatch(fabric_id): + raise FabricError("Fabric 标识无效") + lock = self._locks.setdefault(fabric_id, asyncio.Lock()) + async with lock: + if fabric_id in self._operation_tasks_by_fabric: + raise FabricError("Fabric 操作正在执行", 409, "fabric_operation_busy") + recovery = await self._retained_operation_recovery(fabric_id) + # This drained section contains only local admission writes. + # Cancellation cannot strand a successful CAS handoff between its + # old terminal receipt and new durable background owner. + return await _await_mutation_completion( + self._admit_reconcile_recovery(fabric_id, recovery)) + + async def _admit_reconcile_recovery(self, fabric_id: str, recovery): + guard = None + context = current_execution() + if recovery is not None: + original, desired, before, plan = recovery + guard = await _await_mutation_completion(asyncio.to_thread( + context.coordinator.resolve_and_claim, original, + f"fabric-terminal-v1 operation={original['operation_id']} " + f"plan={plan.digest} status={desired['operation']['status']} " + f"hosts={len(plan.nodes)}; stopped-unit and immutable-spool verified", + execution_id=context.claim.execution_id)) + with (execution_context(context.coordinator, guard) + if guard is not None else nullcontext()): + try: + if guard is not None: + async with self.catalog_mutation_lock(): + current = await asyncio.to_thread( + self._load, "fabric_desired", fabric_id) + if current != before: + raise FabricError("原 Fabric 操作记录已变化", 409, + "fabric_operation_status_invalid") + await _await_mutation_completion(self._persist_operation(desired)) + result = await self._enqueue_reconcile_admitted(fabric_id) + except BaseException: + if guard is not None: + await _await_mutation_completion(asyncio.to_thread( + context.coordinator.retain, guard)) + raise + else: + if guard is not None: + released = await _await_mutation_completion(asyncio.to_thread( + context.coordinator.release, guard)) + if not released: + raise FabricError("Fabric 恢复占用未确认释放", 503, + "fabric_operation_status_invalid") + return result + + async def _retained_operation_recovery(self, fabric_id: str): + """Prepare read-only proof under the original, still intact fence. + + Periodic observation never calls this path. Unknown owners and scopes + remain subject to normal admission; only this exact Fabric background + claim is eligible for explicit atomic ownership transfer. + """ + context = current_execution() + if context is None: + return None + before = await asyncio.to_thread(self._load, "fabric_desired", fabric_id) + if not isinstance(before, dict): + return None + operation = before.get("operation") + if (not isinstance(operation, dict) + or not isinstance(operation.get("id"), str) + or not _OPERATION_ID.fullmatch(operation["id"])): + return None + operation_id = operation["id"] + claims = await asyncio.to_thread(context.coordinator.snapshot) + eligible = [claim for claim in claims.values() if ( + claim["state"] == "uncertain" and claim["kind"] == "background" + and claim["operation_id"] == operation_id + and claim["resources"] == ["catalog:network", f"fabric:{fabric_id}"] + and claim["maintenance"] is False + and claim["metadata"] == {"job_kind": "fabric_operation", "job_id": operation_id})] + if len(eligible) != 1: + return None + original = eligible[0] + if self._safe_operation_projection(operation) is None: + raise FabricError("原 Fabric 操作记录无法确认", 409, + "fabric_operation_status_invalid") + # Do not borrow this recovery proof to resolve a second owner. + for claim in claims.values(): + if claim["token"] == original["token"]: + continue + if claim["maintenance"] or ( + (claim["state"] == "uncertain" + or claim["execution_id"] != context.claim.execution_id) + and (claim["operation_id"] == operation_id + or set(original["resources"]).intersection(claim["resources"])) + ): + return None + desired = json.loads(json.dumps(before)) + plan = self._plan_from_record(desired) + if desired.get("plan_digest") != plan.digest: + raise FabricError("Fabric 计划摘要与持久化记录不一致", 409, + "fabric_operation_status_invalid") + nodes = desired.get("nodes") + keys = desired.get("public_keys") + if (not isinstance(nodes, list) or len(nodes) != len(plan.nodes) + or not isinstance(keys, dict) + or set(keys) != {host.host_id for host in plan.nodes} + or any(not isinstance(node, dict) + or not isinstance(node.get("operation"), dict) + or node["operation"].get("id") != operation_id for node in nodes)): + raise FabricError("原 Fabric 操作记录无法确认", 409, + "fabric_operation_status_invalid") + public_keys = {host_id: validate_wireguard_key(key) for host_id, key in keys.items()} + if len(set(public_keys.values())) != len(public_keys): + raise FabricError("Fabric 公钥记录包含重复身份", 409, "fabric_operation_status_invalid") + node_records = {node.get("host_id"): node for node in nodes} + if set(node_records) != {host.host_id for host in plan.nodes}: + raise FabricError("原 Fabric 节点记录无法确认", 409, "fabric_operation_status_invalid") + if any( + self._safe_operation_projection(node["operation"], node=True) is None + or not self._node_operation_may_have_dispatched(node, operation_id) + for node in nodes + ): + # This migration has authority only over a fully dispatched old + # operation. Missing/unknown phases cannot prove no host mutation. + raise FabricError("原 Fabric 操作尚无完整宿主派发证据", 409, + "fabric_operation_status_invalid") + adapters = {} + for host in plan.nodes: + await self.registry.get(host.host_id) + component = self.registry.components.get(host.host_id) + if component is None: + raise FabricError("Fabric 宿主机组件不存在", 409, + "fabric_operation_status_invalid") + profile = await asyncio.to_thread(component.manager.current) + if (not isinstance(profile, Mapping) or not profile + or profile.get("host_id") not in (None, host.host_id)): + raise FabricError("宿主机稳定标识与原 Fabric 节点不一致", 409, + "fabric_operation_status_invalid") + adapters[host.host_id] = FabricHostAdapter(component.manager, profile) + semaphore = asyncio.Semaphore(FABRIC_PROBE_CONCURRENCY) + + async def terminal_proof(host): + try: + async with semaphore: + proof = await asyncio.to_thread( + adapters[host.host_id].terminal_operation_proof, + operation_id, plan.digest) + except FabricError: + raise FabricError("原 Fabric 宿主任务尚未证明结束,请继续核对原操作", 409, + "fabric_operation_status_invalid", "observe") from None + if (not isinstance(proof, Mapping) or proof.get("operation_id") != operation_id + or proof.get("status") not in {"succeeded", "failed"} + or type(proof.get("finished_at")) is not int): + raise FabricError("原 Fabric 宿主任务终态无效", 409, + "fabric_operation_status_invalid", "observe") + return proof + + # Spool/unit inspection is independent of the host's network manager. + # Full fresh capability and network readback remain in the new worker. + results = await asyncio.gather( + *(terminal_proof(host) for host in plan.nodes), return_exceptions=True) + for result in results: + if isinstance(result, BaseException): + raise result + proofs = {host.host_id: result for host, result in zip(plan.nodes, results)} + self._invalidate(desired) + for host in plan.nodes: + node = node_records[host.host_id] + proof = proofs[host.host_id] + self._record_remote_operation_status(node, proof) + # A stopped old runner proves safety to repair, not current + # network convergence. Leave the gate closed and let the new + # background operation perform fresh observation and any apply. + self._update_node_operation( + node, status="failed", phase="observe", checkpoint="final_readback", + reason="readback_failed") + if not node.get("last_error_code"): + self._record_node_error(node, "fabric_readback_mismatch", failure_phase="observe") + self._update_root_operation(desired, "failed") + desired["reconciled_operation"] = { + "operation": dict(desired["operation"]), "plan_digest": plan.digest, + "claim_token": original["token"], + "reason": "terminal_hosts_require_fresh_reconcile", + "nodes": [{"host_id": node["host_id"], "operation": dict(node["operation"]), + "remote_status": proofs[node["host_id"]]["status"], + "last_error_code": node.get("last_error_code")} + for node in nodes], + } + return original, desired, before, plan + + async def _enqueue_reconcile_admitted(self, fabric_id: str) -> dict[str, Any]: """Queue an explicit fixed-plan reconciliation without HTTP waiting.""" if not _FABRIC_KEY.fullmatch(fabric_id): raise FabricError("Fabric 标识无效") @@ -10443,133 +10748,131 @@ class FabricCoordinator: resume_status_only = False resume_kind = "reconcile" identity_bootstrap = False - lock = self._locks.setdefault(fabric_id, asyncio.Lock()) - async with lock: - # Lock order is always per-Fabric -> shared catalog. Close the - # authoritative gate and admit the operation while holding the - # same lock used by VPC catalog consumers. - async with self.catalog_mutation_lock(): - desired, plan = await _await_mutation_completion( - self._begin_reconcile(fabric_id)) - old_operation = desired.get("operation") - if (isinstance(old_operation, Mapping) - and old_operation.get("status") in { - "queued", "running", "interrupted", "uncertain"} - and isinstance(old_operation.get("id"), str) - and _OPERATION_ID.fullmatch( - old_operation["id"]) is not None): - unsettled = any( - isinstance(node, Mapping) - and isinstance(node.get("operation"), Mapping) - and node["operation"].get("id") - == old_operation["id"] - and node["operation"].get("status") in { - "pending", "running", "uncertain"} - for node in desired.get("nodes", []) - ) - # A queued/running durable row without a task owned by - # this process is necessarily a restart join. Periodic - # readback may have changed the recovered root from - # interrupted back to running after observing the same - # remote queued job. Explicit reconcile must therefore - # retain the exact id and enter the guarded queued-gap - # recovery instead of returning busy forever. The task - # registry check above remains the authoritative in- - # process concurrency guard. - if (old_operation.get("status") in {"queued", "running"} - or unsettled): - operation_id = old_operation["id"] - resume_existing = True - resume_status_only = ( - self._operation_may_have_dispatched( - desired, operation_id)) - old_kind = old_operation.get("kind") - if old_kind not in FABRIC_OPERATION_KINDS: - raise FabricError( - "Fabric 操作记录损坏", 500, - "fabric_record_corrupt") - resume_kind = old_kind - elif (isinstance(old_operation, Mapping) - and old_operation.get("status") in { - "queued", "running"}): - # Never replace an apparently live attempt whose exact - # operation identity cannot be proven. - raise FabricError( - "Fabric 操作记录损坏", 500, - "fabric_record_corrupt") + # Lock order is always per-Fabric -> shared catalog. Close the + # authoritative gate and admit the operation while holding the + # same lock used by VPC catalog consumers. + async with self.catalog_mutation_lock(): + desired, plan = await _await_mutation_completion( + self._begin_reconcile(fabric_id)) + old_operation = desired.get("operation") + if (isinstance(old_operation, Mapping) + and old_operation.get("status") in { + "queued", "running", "interrupted", "uncertain"} + and isinstance(old_operation.get("id"), str) + and _OPERATION_ID.fullmatch( + old_operation["id"]) is not None): + unsettled = any( + isinstance(node, Mapping) + and isinstance(node.get("operation"), Mapping) + and node["operation"].get("id") + == old_operation["id"] + and node["operation"].get("status") in { + "pending", "running", "uncertain"} + for node in desired.get("nodes", []) + ) + # A queued/running durable row without a task owned by + # this process is necessarily a restart join. Periodic + # readback may have changed the recovered root from + # interrupted back to running after observing the same + # remote queued job. Explicit reconcile must therefore + # retain the exact id and enter the guarded queued-gap + # recovery instead of returning busy forever. The task + # registry check above remains the authoritative in- + # process concurrency guard. + if (old_operation.get("status") in {"queued", "running"} + or unsettled): + operation_id = old_operation["id"] + resume_existing = True + resume_status_only = ( + self._operation_may_have_dispatched( + desired, operation_id)) + old_kind = old_operation.get("kind") + if old_kind not in FABRIC_OPERATION_KINDS: + raise FabricError( + "Fabric 操作记录损坏", 500, + "fabric_record_corrupt") + resume_kind = old_kind + elif (isinstance(old_operation, Mapping) + and old_operation.get("status") in { + "queued", "running"}): + # Never replace an apparently live attempt whose exact + # operation identity cannot be proven. + raise FabricError( + "Fabric 操作记录损坏", 500, + "fabric_record_corrupt") - raw_nodes = desired.get("nodes") - if not isinstance(raw_nodes, list): - raise FabricError( - "Fabric 节点记录损坏", 500, - "fabric_record_corrupt") - public_keys = desired.get("public_keys") - validated_keys: dict[str, str] | None = None - if (isinstance(public_keys, dict) - and set(public_keys) == { - node.host_id for node in plan.nodes}): - try: - validated_keys = { - host_id: validate_wireguard_key(key) - for host_id, key in public_keys.items() - } - except FabricError: - validated_keys = None - if (validated_keys is not None - and len(set(validated_keys.values())) - != len(validated_keys)): - validated_keys = None - - # A failed attempt with no durable apply-phase node proves - # that no host unit could have been accepted. Explicit - # reconcile may safely resume identity collection for the - # fixed desired plan; all other incomplete key records remain - # fail-closed corruption. - identity_bootstrap = bool( - validated_keys is None - and isinstance(old_operation, Mapping) - and not self._operation_may_have_dispatched( - desired, old_operation.get("id")) - and (resume_existing - or old_operation.get("status") == "failed")) - if validated_keys is None and not identity_bootstrap: - raise FabricError( - "Fabric 公钥记录损坏", 500, - "fabric_record_corrupt") - - if resume_existing: - # Preserve the exact operation id. Before the durable - # apply fence it is safe to continue identity/coordinator - # work under that id; after the fence this admission is - # strictly status/readback-only. - self._update_root_operation(desired, "running") - else: - now = int(time.time()) - desired["operation"] = self._new_root_operation( - operation_id, "reconcile", now) - if identity_bootstrap: - desired["public_keys"] = {} - for node in raw_nodes: - if not isinstance(node, dict): - raise FabricError( - "Fabric 节点记录损坏", 500, - "fabric_record_corrupt") - node["operation"] = self._new_node_operation( - operation_id, - "identity" if identity_bootstrap else "observe", - now) - desired["state"] = "applying" - desired["next_stage_allowed"] = False - await _await_mutation_completion( - self._persist_operation(desired)) - await self._track_operation_task( - fabric_id, operation_id, - (self._observe_only( - fabric_id, allow_tracked=True, recover_queued=True) - if resume_status_only else - self._run_enqueued_operation( - fabric_id, operation_id, - resume_kind if resume_existing else "reconcile"))) + raw_nodes = desired.get("nodes") + if not isinstance(raw_nodes, list): + raise FabricError( + "Fabric 节点记录损坏", 500, + "fabric_record_corrupt") + public_keys = desired.get("public_keys") + validated_keys: dict[str, str] | None = None + if (isinstance(public_keys, dict) + and set(public_keys) == { + node.host_id for node in plan.nodes}): + try: + validated_keys = { + host_id: validate_wireguard_key(key) + for host_id, key in public_keys.items() + } + except FabricError: + validated_keys = None + if (validated_keys is not None + and len(set(validated_keys.values())) + != len(validated_keys)): + validated_keys = None + + # A failed attempt with no durable apply-phase node proves + # that no host unit could have been accepted. Explicit + # reconcile may safely resume identity collection for the + # fixed desired plan; all other incomplete key records remain + # fail-closed corruption. + identity_bootstrap = bool( + validated_keys is None + and isinstance(old_operation, Mapping) + and not self._operation_may_have_dispatched( + desired, old_operation.get("id")) + and (resume_existing + or old_operation.get("status") == "failed")) + if validated_keys is None and not identity_bootstrap: + raise FabricError( + "Fabric 公钥记录损坏", 500, + "fabric_record_corrupt") + + if resume_existing: + # Preserve the exact operation id. Before the durable + # apply fence it is safe to continue identity/coordinator + # work under that id; after the fence this admission is + # strictly status/readback-only. + self._update_root_operation(desired, "running") + else: + now = int(time.time()) + desired["operation"] = self._new_root_operation( + operation_id, "reconcile", now) + if identity_bootstrap: + desired["public_keys"] = {} + for node in raw_nodes: + if not isinstance(node, dict): + raise FabricError( + "Fabric 节点记录损坏", 500, + "fabric_record_corrupt") + node["operation"] = self._new_node_operation( + operation_id, + "identity" if identity_bootstrap else "observe", + now) + desired["state"] = "applying" + desired["next_stage_allowed"] = False + await _await_mutation_completion( + self._persist_operation(desired)) + await self._track_operation_task( + fabric_id, operation_id, + (self._observe_only( + fabric_id, allow_tracked=True, recover_queued=True) + if resume_status_only else + self._run_enqueued_operation( + fabric_id, operation_id, + resume_kind if resume_existing else "reconcile"))) return self._public(desired) async def _persist_operation(self, desired: dict[str, Any]) -> None: @@ -11991,6 +12294,11 @@ class FabricCoordinator: # Expected host/configuration failures are already projected # through fixed error codes and operation checkpoints. continue + except CoordinationClaimRejected: + # A retained owner deliberately keeps the gate closed until + # explicit exact-operation recovery proves its host jobs have + # stopped. It is an admission conflict, not an internal bug. + continue except Exception: # As above, omit the original exception because third-party # exceptions may carry raw remote output. The validated id diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index c0d87376..2525f06b 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -51,7 +51,8 @@ from .engine_security import (EngineAuthError, authenticate, decode_secret, from .fabric import FabricCoordinator, FabricError from .libvirt_rpc import EXECUTION_ERROR_CODES as LIBVIRT_EXECUTION_ERROR_CODES from .coordination import ( - DurableCoordinator, CoordinationError, current_execution, execution_context, + DurableCoordinator, CoordinationError, CoordinationClaimRejected, + current_execution, execution_context, configure_default_coordinator, update_current_claim, _uncertain_exception, connection_failure_before_write, execution_failure_before_write, mark_execution_unknown, @@ -871,14 +872,27 @@ class EngineApplication: method == "PUT" and resource == "host" or method == "POST" and resource in {"host/test", "host/scan"})) or ( method == "POST" and path == "/internal/v1/hosts/scan") + fabric_admission_request = bool(method == "POST" and re.fullmatch( + r"/internal/v1/fabric/(?:apply|[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile)", path)) async def execute(): await asyncio.to_thread(self._fence_legacy_operations) await self._settle_update_claims() - claim = await asyncio.to_thread( - self.coordinator.claim, - operation_id=op_id if isinstance(op_id, str) and OP_ID.fullmatch(op_id) else None, - resources=() if cancel_queued_snapshot else resources, kind="request") + try: + claim = await asyncio.to_thread( + self.coordinator.claim, + operation_id=op_id if isinstance(op_id, str) and OP_ID.fullmatch(op_id) else None, + resources=() if cancel_queued_snapshot else resources, kind="request") + except CoordinationClaimRejected as exc: + if (type(exc) is not CoordinationClaimRejected or not fabric_admission_request + or exc.error_code not in {"coordination_busy", "coordination_uncertain"}): + raise + messages = [] + async def capture_rejection(message): + messages.append(message) + await json_response(capture_rejection, 409, { + "error": str(exc), "error_code": exc.error_code}) + return messages dispatch_started = False messages = [] async def capture(message): @@ -947,6 +961,24 @@ class EngineApplication: else: await asyncio.to_thread(self.coordinator.release, claim) except BaseException as exc: + # A Fabric catalog claim rejected before any child owner + # exists is a definite non-admission, even when the older + # owner's outcome is uncertain. Never infer this from an + # error code shared by actual storage/transport failures. + context = current_execution() + if (type(exc) is CoordinationClaimRejected + and fabric_admission_request + and exc.error_code in {"coordination_busy", "coordination_uncertain"} + and not context.remote_write_started.is_set() + and not context.execution_unknown.is_set()): + owners = await asyncio.to_thread(self.coordinator.snapshot) + if not any(row["execution_id"] == claim.execution_id + and row["token"] != claim.token + for row in owners.values()): + if await asyncio.to_thread(self.coordinator.release, claim): + await json_response(capture, 409, { + "error": str(exc), "error_code": exc.error_code}) + return messages from .libvirt_service import VMError not_started = False try: diff --git a/web/app/Middleware/CloudApi.php b/web/app/Middleware/CloudApi.php index 38eaba34..52e25747 100644 --- a/web/app/Middleware/CloudApi.php +++ b/web/app/Middleware/CloudApi.php @@ -53,7 +53,18 @@ final class CloudApi $response ); } catch (CloudError $exception) { - return $response->withJson(['error' => $exception->getMessage(), 'detail' => $exception->getMessage()], $exception->httpStatus) + $payload = ['error' => $exception->getMessage(), 'detail' => $exception->getMessage()]; + // A reviewed Fabric 409 proves this request was rejected before + // admission. Keep its fixed code without exposing arbitrary + // Engine diagnostics or changing other Cloud API error shapes. + if (strtoupper($request->getMethod()) === 'POST' + && preg_match('#^/api/v1/admin/fabric/regions/[1-9][0-9]*/(?:apply|reconcile)$#D', $path ?? '') === 1 + && $exception->httpStatus === 409 + && in_array($exception->errorCode, + ['coordination_busy', 'coordination_uncertain'], true)) { + $payload['error_code'] = $exception->errorCode; + } + return $response->withJson($payload, $exception->httpStatus) ->withHeader('Cache-Control', 'no-store'); } catch (\Throwable $exception) { error_log('Cloud API unavailable: ' . get_class($exception)); diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index 65205310..776bb63e 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -100,6 +100,15 @@ final class CloudEngineClient $response->getStatusCode(), $expectedOperationId)) { throw new CloudError('实例操作正在排队,请稍后核对', 409, 'coordination_busy'); } + $fabricAdmissionCode = self::fabricAdmissionError( + $method, $signedPath, $result, $response->getStatusCode() + ); + if ($fabricAdmissionCode !== null) { + $message = $fabricAdmissionCode === 'coordination_busy' + ? '协调资源正由原任务占用,本次请求未接纳;请先核对原操作。' + : '原任务结果尚未确认,本次请求未接纳;请先恢复原操作。'; + throw new CloudError($message, 409, $fabricAdmissionCode); + } $vpcApplyCoordinationCode = self::vpcApplyCoordinationError( $method, $signedPath, $result, $response->getStatusCode(), $expectedOperationId @@ -152,6 +161,22 @@ final class CloudEngineClient return $result; } + private static function fabricAdmissionError(string $method, ?string $path, + array $result, int $status): ?string + { + // Only a bare 409 from these reviewed mutation routes proves that + // this request was not admitted. The same code on 5xx or a response + // carrying an operation/outcome remains an uncertain write result. + if ($status !== 409 || strtoupper($method) !== 'POST' || $path === null + || preg_match('#^/internal/v1/fabric/(?:apply|[A-Za-z0-9][A-Za-z0-9_.-]{0,63}/reconcile)$#D', $path) !== 1 + || array_diff(array_keys($result), ['error', 'error_code']) !== []) { + return null; + } + $code = $result['error_code'] ?? null; + return in_array($code, ['coordination_busy', 'coordination_uncertain'], true) + ? $code : null; + } + private static function isPowerAdmissionBusy(string $method, ?string $path, array $result, int $status, ?string $expectedOperationId): bool { diff --git a/web/public/cloud/admin-fabric.js b/web/public/cloud/admin-fabric.js index 405c03b1..c081ddb8 100644 --- a/web/public/cloud/admin-fabric.js +++ b/web/public/cloud/admin-fabric.js @@ -18,6 +18,10 @@ const state = { const receiptRecoveryInterval = 5000; const receiptRecoveryMaxReads = 36; const receiptRecoveryMaxDuration = 180000; +const fabricAdmissionMessages = { + coordination_busy: '协调资源正由原任务占用,本次请求未接纳;请先核对原操作。', + coordination_uncertain: '原任务结果尚未确认,本次请求未接纳;请先恢复原操作。', +}; const workflowStages = new Set([ 'engine_unavailable', 'no_hosts', 'needs_probe', 'blocked', @@ -433,9 +437,14 @@ async function api(path, options = {}) { try { payload = await response.json(); } catch (ignore) {} if (!response.ok) { const message = payload && (payload.detail || payload.error || payload.message); - const failure = new Error(typeof message === 'string' && message - ? message : '请求失败 (' + response.status + ')'); + const admissionCode = method === 'POST' && response.status === 409 + && /^\/api\/v1\/admin\/fabric\/regions\/[1-9][0-9]*\/(apply|reconcile)$/.test(path) + && payload && Object.prototype.hasOwnProperty.call(fabricAdmissionMessages, payload.error_code) + ? payload.error_code : null; + const failure = new Error(admissionCode ? fabricAdmissionMessages[admissionCode] + : typeof message === 'string' && message ? message : '请求失败 (' + response.status + ')'); failure.httpStatus = response.status; + if (admissionCode) failure.errorCode = admissionCode; throw failure; } return payload; @@ -590,9 +599,11 @@ function renderWizardSteps(region) { }; } const unknown = region && state.outcomeUnknownRegionIds.has(regionIdOf(region)); + const recoveryDiagnostic = region && originalReceiptRecoveryDiagnostic( + state.receiptRecoveriesByRegion.get(regionIdOf(region)), trackedOperation); byId('fabric-wizard-stage').textContent = unknown ? '操作结果尚未确认' : copy.title; byId('fabric-wizard-stage-help').textContent = unknown - ? '上次写操作失去响应;页面只读取状态,不会重复提交。尚未确认本轮 operation 时所有写操作保持关闭;如只读恢复停止,请刷新页面查看权威状态。' + ? recoveryDiagnostic || '上次写操作失去响应;页面只读取状态,不会重复提交。尚未确认本轮 operation 时所有写操作保持关闭;如只读恢复停止,请刷新页面查看权威状态。' : copy.help; const order = ['select', 'probe', 'apply', 'reconcile']; @@ -920,6 +931,14 @@ function receiptRecoveryMatches(recovery, observed) { && (!previous || previous.updated_at === null || observed.started_at >= previous.updated_at); } +function originalReceiptRecoveryDiagnostic(recovery, observed) { + if (!recovery || !recovery.previousOperation || !observed + || observed.id !== recovery.previousOperation.id + || !['uncertain', 'interrupted'].includes(observed.status) + || !receiptRecoveryMatches(recovery, observed)) return ''; + return '原 operation ' + observed.id + ' ' + operationStatusLabel(observed.status) + + ',仍待恢复;尚未确认本次请求是否接纳,页面继续只读核对,不会重复提交。'; +} function adoptRegionOperations() { state.regions.forEach(region => { const regionId = regionIdOf(region); @@ -1050,8 +1069,12 @@ async function pollOperations() { notice(regionName(selectedRecovery.region) + ':只读发现 operation ' + selectedRecovery.operation.id + ',正在继续读取同一任务;不会重复提交。'); } else if (state.receiptRecoveriesByRegion.has(state.selectedRegionId)) { + const recovery = state.receiptRecoveriesByRegion.get(state.selectedRegionId); + const region = selectedRegion(); + const diagnostic = originalReceiptRecoveryDiagnostic( + recovery, operationOf(region && region.operation)); notice(regionName(selectedRegion()) - + ':尚未确认本轮 operation;正在只读恢复接纳结果,不会把旧任务状态当成本轮结果,也不会重复提交。', 'error'); + + ':' + (diagnostic || '尚未确认本轮 operation;正在只读恢复接纳结果,不会把旧任务状态当成本轮结果,也不会重复提交。'), 'error'); } else if (state.operationsByRegion.has(state.selectedRegionId)) { const operation = state.operationsByRegion.get(state.selectedRegionId); notice(regionName(selectedRegion()) + ':operation ' + operation.id @@ -1225,7 +1248,13 @@ async function runRegionAction(action) { startReceiptRecovery(regionId, action, previousOperation); if (state.selectedRegionId === regionId) renderSelectedRegion(); } - notice(originName + ':' + (error.message || '操作未完成,请重新核对。'), 'error'); + const previousPending = error && error.httpStatus === 409 + && Object.prototype.hasOwnProperty.call(fabricAdmissionMessages, error.errorCode) + && previousOperation && previousOperation.id + && ['queued', 'running', 'uncertain', 'interrupted'].includes(previousOperation.status) + ? ' 原 operation ' + previousOperation.id + ' 仍待恢复。' : ''; + notice(originName + ':' + (error.message || '操作未完成,请重新核对。') + + previousPending, 'error'); } finally { state.pendingAction = ''; syncControls(); diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 2b158777..d8d3e0ee 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026101403' scope='global'} +{assign var=venix_asset_version value='2026101404' scope='global'} diff --git a/web/tests/AdminFabricWizardBehaviorTest.mjs b/web/tests/AdminFabricWizardBehaviorTest.mjs index bc3dd438..afc46a34 100644 --- a/web/tests/AdminFabricWizardBehaviorTest.mjs +++ b/web/tests/AdminFabricWizardBehaviorTest.mjs @@ -546,6 +546,61 @@ function fixture(initialRegions, canManage = true) { assert.equal(test.requests.length, 2, 'A definitive 409 must not start lost-receipt GET recovery.'); } +// A reviewed coordination 409 is a definite rejection before admission. +// Preserve the original task as evidence without inventing a new receipt. +for (const code of ['coordination_busy', 'coordination_uncertain']) { + const old = operation('uncertain', 'reconcile'); + const initial = region(1, 'needs_reconcile', { operation: old }); + const test = fixture([initial]); + await test.respond(test.requests[0], { regions: [initial], engine_available: true }); + test.nodes['fabric-reconcile'].dispatch('click'); await flush(); + await test.respond(test.requests[1], { + error_code: code, detail: 'Traceback private_key=/root/private', + }, 409); + assert.match(test.nodes['fabric-notice'].textContent, /本次请求未接纳/); + assert.match(test.nodes['fabric-notice'].textContent, new RegExp('原 operation ' + old.id)); + assert.doesNotMatch(test.nodes['fabric-notice'].textContent, /Traceback|private_key|\/root/); + assert.doesNotMatch(test.nodes['fabric-wizard-stage'].textContent, /尚未确认/); + assert.equal(test.nodes['fabric-reconcile'].disabled, false, + 'A proved admission rejection keeps the server-owned recovery workflow.'); + await test.advance(10000); + assert.equal(test.requests.length, 2, + 'A proved admission rejection must not invent a lost-receipt poller.'); + assert.equal(test.requests.filter(request => request.options.method === 'POST').length, 1); +} + +// A coordination code on 5xx is not a pre-admission rejection proof. A +// matching old uncertain/interrupted task stays unknown, with its exact ID +// shown as diagnostic evidence rather than a newly accepted operation. +for (const status of [502, 503]) { + const old = operation('uncertain', 'reconcile'); + const initial = region(1, 'needs_reconcile', { operation: old }); + const test = fixture([initial]); + await test.respond(test.requests[0], { regions: [initial], engine_available: true }); + test.nodes['fabric-reconcile'].dispatch('click'); await flush(); + await test.respond(test.requests[1], { + error_code: 'coordination_uncertain', detail: 'coordination rejected', + }, status); + assert.match(test.nodes['fabric-notice'].textContent, /结果尚未确认/); + assert.doesNotMatch(test.nodes['fabric-notice'].textContent, /本次请求未接纳/); + for (const oldStatus of ['uncertain', 'interrupted']) { + await test.advance(5000); + await test.respond(test.requests.at(-1), { + regions: [region(1, 'needs_reconcile', { + operation: { ...old, status: oldStatus, updated_at: old.updated_at + 20 }, + })], engine_available: true, + }); + assert.match(test.nodes['fabric-notice'].textContent, + new RegExp('原 operation ' + old.id + '.*仍待恢复')); + assert.match(test.nodes['fabric-notice'].textContent, /尚未确认本次请求是否接纳/); + assert.doesNotMatch(test.nodes['fabric-notice'].textContent, /只读发现 operation|已受理/); + assert.match(test.nodes['fabric-wizard-stage-help'].textContent, + new RegExp('原 operation ' + old.id)); + assert.equal(test.actions.every(button => button.disabled), true); + } + assert.equal(test.requests.filter(request => request.options.method === 'POST').length, 1); +} + // Unknown outcomes are isolated by region. A later successful action for a // different region must not clear the first region's stale-write guard. { diff --git a/web/tests/CloudFabricAdmissionErrorContractTest.php b/web/tests/CloudFabricAdmissionErrorContractTest.php new file mode 100644 index 00000000..65b89ed7 --- /dev/null +++ b/web/tests/CloudFabricAdmissionErrorContractTest.php @@ -0,0 +1,318 @@ + 1, 'is_admin' => true, 'can_manage' => true]; + } + } +} + +namespace GuzzleHttp\Exception { + class ConnectException extends \RuntimeException + { + } +} + +namespace GuzzleHttp { + final class FabricAdmissionTestResponse + { + public function __construct(private int $status, private string $body) + { + } + + public function getStatusCode(): int + { + return $this->status; + } + + public function getBody(): string + { + return $this->body; + } + } + + class Client + { + public function __construct(private array $responses = []) + { + } + + public function request(string $method, string $url, array $options): FabricAdmissionTestResponse + { + $response = array_shift($this->responses); + if ($response instanceof \Throwable) { + throw $response; + } + if (!$response instanceof FabricAdmissionTestResponse) { + throw new \RuntimeException('No fake Fabric engine response was queued.'); + } + return $response; + } + } +} + +namespace { +use App\Middleware\CloudApi; +use App\Services\Cloud\CloudEngineClient; +use App\Services\Cloud\CloudError; +use GuzzleHttp\Client; +use GuzzleHttp\Exception\ConnectException; +use GuzzleHttp\FabricAdmissionTestResponse; + +require dirname(__DIR__) . '/app/Services/Cloud/CloudError.php'; +require dirname(__DIR__) . '/app/Services/Cloud/CloudEngineClient.php'; +require dirname(__DIR__) . '/app/Middleware/CloudApi.php'; + +$assertions = 0; +$failures = []; +$assert = static function ($condition, string $message) use (&$assertions, &$failures): void { + $assertions++; + if (!$condition) { + $failures[] = $message; + } +}; +$messages = [ + 'coordination_busy' => '协调资源正由原任务占用,本次请求未接纳;请先核对原操作。', + 'coordination_uncertain' => '原任务结果尚未确认,本次请求未接纳;请先恢复原操作。', +]; +$actor = ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]; +$body = static fn ($code): array => [ + 'error' => 'Traceback: host=/root/private key=secret', 'error_code' => $code, +]; +$request = static function (string $method, string $path, int $status, + array $payload) use ($actor): CloudError { + $client = new CloudEngineClient(new Client([ + new FabricAdmissionTestResponse($status, json_encode($payload, JSON_THROW_ON_ERROR)), + ])); + try { + // Fabric admission has no caller-generated operation ID or receipt. + $client->request($method, $path, $actor, []); + } catch (CloudError $error) { + return $error; + } + throw new RuntimeException('The fake Fabric failure did not raise CloudError.'); +}; + +// Only the exact mutation endpoints and bare 409 rejection cross this boundary. +foreach (['/internal/v1/fabric/apply', '/internal/v1/fabric/hk-1_2.3/reconcile', + '/internal/v1/fabric/1/reconcile', '/internal/v1/fabric/' . str_repeat('a', 64) . '/reconcile'] as $path) { + foreach ($messages as $code => $message) { + $error = $request('POST', $path, 409, $body($code)); + $assert($error->httpStatus === 409 && $error->errorCode === $code + && $error->getMessage() === $message + && !str_contains($error->getMessage(), '/root') + && !str_contains($error->getMessage(), 'secret'), + 'A bare Fabric admission rejection lost its fixed status, code or message: ' . $code . ' ' . $path); + } +} + +$invalidRoutes = [ + ['GET', '/internal/v1/fabric/apply'], + ['PUT', '/internal/v1/fabric/apply'], + ['DELETE', '/internal/v1/fabric/hk/reconcile'], + ['POST', '/internal/v1/fabric'], + ['POST', '/internal/v1/fabric/probe'], + ['POST', '/internal/v1/fabric/hk'], + ['POST', '/internal/v1/fabric/hk/reconcile/extra'], + ['POST', '/internal/v1/fabric/hk/reconcile?unexpected=true'], + ['POST', '/internal/v1/fabric/apply?unexpected=true'], + ['POST', '/internal/v1/fabric//reconcile'], + ['POST', '/internal/v1/fabric/-hk/reconcile'], + ['POST', '/internal/v1/fabric/' . str_repeat('a', 65) . '/reconcile'], + ['POST', '/internal/v1/vpcs/apply'], +]; +foreach ($invalidRoutes as [$method, $path]) { + foreach (array_keys($messages) as $code) { + $error = $request($method, $path, 409, $body($code)); + $assert($error->httpStatus === 502 && $error->errorCode === null + && $error->getMessage() === '虚拟化操作未成功(引擎状态 409)', + 'A method or path outside Fabric admission received a definitive rejection: ' . $method . ' ' . $path); + } +} + +foreach ([400, 403, 404, 422, 500, 502, 503, 504] as $status) { + foreach (array_keys($messages) as $code) { + foreach (['/internal/v1/fabric/apply', '/internal/v1/fabric/hk/reconcile'] as $path) { + $error = $request('POST', $path, $status, $body($code)); + $assert($error->httpStatus === ($status === 404 ? 404 : 502) + && $error->errorCode === null + && $error->getMessage() === '虚拟化操作未成功(引擎状态 ' . $status . ')', + 'An unreviewed HTTP status became a Fabric admission rejection: ' . $status . ' ' . $code); + } + } +} +foreach (['coordination_unavailable', 'fabric_unknown_failure', '', null, ['coordination_busy']] as $code) { + $error = $request('POST', '/internal/v1/fabric/apply', 409, $body($code)); + $assert($error->httpStatus === 502 && $error->errorCode === null, + 'An unknown or malformed code crossed the Fabric admission allowlist.'); +} +foreach ([ + ['operation_id' => str_repeat('b', 32)], + ['operation' => ['id' => str_repeat('b', 32), 'status' => 'uncertain']], + ['status' => 'uncertain'], + ['safe_to_refund' => false], + ['result' => ['command_accepted' => true]], + ['mutation_started' => true], + ['mutation_started' => false], + ['execution_outcome' => 'unknown'], + ['execution_outcome' => 'not_dispatched'], + ['failure_phase' => 'apply'], + ['extra' => null], +] as $extra) { + foreach (array_keys($messages) as $code) { + foreach (['/internal/v1/fabric/apply', '/internal/v1/fabric/hk/reconcile'] as $path) { + $error = $request('POST', $path, 409, array_replace($body($code), $extra)); + $assert($error->httpStatus === 502 && $error->errorCode === null, + 'An outcome or extra field was mistaken for a bare Fabric admission rejection: ' . array_key_first($extra)); + } + } +} + +foreach ([ + new Client([new ConnectException('connection lost /root/private secret')]), + new Client([new RuntimeException('transport failed /root/private secret')]), + new Client([new FabricAdmissionTestResponse(502, 'bad gateway')]), + new Client([new FabricAdmissionTestResponse(503, 'null')]), +] as $transportClient) { + try { + (new CloudEngineClient($transportClient))->request( + 'POST', '/internal/v1/fabric/hk/reconcile', $actor, [] + ); + throw new RuntimeException('The fake Fabric transport failure did not raise CloudError.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 502 && $error->errorCode === 'engine_transport_uncertain' + && !str_contains($error->getMessage(), '/root') + && !str_contains($error->getMessage(), 'secret'), + 'A real Fabric transport failure lost its unknown-outcome boundary.'); + } +} + +final class FabricAdmissionApiRequest +{ + public function __construct(private string $method, private string $path) + { + } + + public function getMethod(): string { return $this->method; } + public function getUri() { return new class($this->path) { + public function __construct(private string $path) {} + public function getPath(): string { return $this->path; } + }; } + public function getHeaderLine(string $name): string { return ''; } + public function getQueryParams(): array { return []; } + public function withHeader(string $name, string $value): self { return clone $this; } + public function withAttribute(string $name, $value): self { return clone $this; } +} + +final class FabricAdmissionApiResponse +{ + public array $payload = []; + public int $status = 200; + public array $headers = []; + + public function withJson(array $payload, int $status): self + { + $response = clone $this; + $response->payload = $payload; + $response->status = $status; + return $response; + } + + public function withHeader(string $name, string $value): self + { + $response = clone $this; + $response->headers[$name] = $value; + return $response; + } +} + +$middleware = static function (string $method, string $path, int $status, + ?string $code) use ($messages): FabricAdmissionApiResponse { + $error = new CloudError($messages[$code ?? ''] ?? '固定安全错误信息', $status, $code); + return (new CloudApi())( + new FabricAdmissionApiRequest($method, $path), new FabricAdmissionApiResponse(), + static function ($request, $response) use ($error) { throw $error; } + ); +}; +foreach (['/api/v1/admin/fabric/regions/1/apply', '/api/v1/admin/fabric/regions/42/reconcile'] as $path) { + foreach ($messages as $code => $message) { + $response = $middleware('POST', $path, 409, $code); + $assert($response->status === 409 && $response->payload === [ + 'error' => $message, 'detail' => $message, 'error_code' => $code, + ] && ($response->headers['Cache-Control'] ?? null) === 'no-store', + 'The exact Fabric route lost its safe admission-code projection: ' . $code . ' ' . $path); + } +} +foreach ([ + ['GET', '/api/v1/admin/fabric/regions/1/apply', 409, 'coordination_busy'], + ['PUT', '/api/v1/admin/fabric/regions/1/reconcile', 409, 'coordination_uncertain'], + ['POST', '/api/v1/admin/fabric/regions/0/reconcile', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric/regions/01/reconcile', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric/regions/1/probe', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric/regions/1/reconcile/extra', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric/regions/hk/reconcile', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric', 409, 'coordination_busy'], + ['POST', '/api/v1/vpcs/' . str_repeat('a', 32) . '/reconcile', 409, 'coordination_busy'], + ['POST', '/api/v1/admin/fabric/regions/1/apply', 503, 'coordination_uncertain'], + ['POST', '/api/v1/admin/fabric/regions/1/apply', 502, 'engine_transport_uncertain'], + ['POST', '/api/v1/admin/fabric/regions/1/apply', 409, 'coordination_unavailable'], + ['POST', '/api/v1/admin/fabric/regions/1/apply', 409, 'fabric_operation_busy'], + ['POST', '/api/v1/admin/fabric/regions/1/apply', 409, null], +] as [$method, $path, $status, $code]) { + $response = $middleware($method, $path, $status, $code); + $message = $messages[$code ?? ''] ?? '固定安全错误信息'; + $assert($response->status === $status + && $response->payload === ['error' => $message, 'detail' => $message] + && ($response->headers['Cache-Control'] ?? null) === 'no-store', + 'An unrelated CloudApi error leaked a Fabric admission machine code: ' . $method . ' ' . $path); +} + +if ($failures !== []) { + throw new RuntimeException(implode("\n", $failures)); +} +echo 'Cloud Fabric admission error contracts passed (' . $assertions . " assertions).\n"; +}