From e55b26f1fd31fc2304c56d8a8f7841cffdf1f871 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 13 Oct 2026 17:02:23 +0800 Subject: [PATCH] fix(engine): release proven undispatched execution claims --- docs/ENGINE_COORDINATION.md | 34 +- docs/VALIDATION.md | 31 ++ .../test_background_prewrite_admission.py | 315 ++++++++++++++++++ engine/tests/test_engine_boundary.py | 8 + engine/tests/test_host_service_scope.py | 2 +- .../tests/test_network_prewrite_admission.py | 136 ++++++++ engine/tests/test_prewrite_admission.py | 193 +++++++++++ .../tests/test_prewrite_execution_boundary.py | 175 ++++++++++ engine/virsh_panel/app.py | 171 +++++++++- engine/virsh_panel/coordination.py | 128 ++++++- engine/virsh_panel/durable_jobs.py | 21 +- engine/virsh_panel/internal.py | 106 +++++- engine/virsh_panel/libvirt_rpc.py | 9 +- engine/virsh_panel/libvirt_service.py | 5 +- web/app/Services/Cloud/CloudWorkerService.php | 40 ++- web/tests/CloudPowerReceiptWorkflowTest.php | 93 +++++- .../CloudPrewriteFailureWorkflowTest.php | 167 ++++++++++ 17 files changed, 1581 insertions(+), 53 deletions(-) create mode 100644 engine/tests/test_background_prewrite_admission.py create mode 100644 engine/tests/test_network_prewrite_admission.py create mode 100644 engine/tests/test_prewrite_admission.py create mode 100644 engine/tests/test_prewrite_execution_boundary.py create mode 100644 web/tests/CloudPrewriteFailureWorkflowTest.php diff --git a/docs/ENGINE_COORDINATION.md b/docs/ENGINE_COORDINATION.md index 8492a955..adc34dc0 100644 --- a/docs/ENGINE_COORDINATION.md +++ b/docs/ENGINE_COORDINATION.md @@ -44,6 +44,37 @@ HTTP 请求的实际执行由受跟踪任务持有,调用者取消只停止等待;原执行继续保存结果后再释放资源。共享停机预算耗尽时保留未完成占用。普通请求的占用覆盖最终 operation 落库,不能在 libvirt 返回后、结果尚未保存时提前放行。 +### 执行前失败与准入拒绝(2026-10-13) + +不能用 HTTP 5xx 判断是否执行过写入。当前 VM 请求、嵌套资源锁、RPC 与后台线程 +共享本次执行的写入和未知结果证据;仅对远端修改全部经过 RPC 的 VM 路径启用。 +`execution_outcome` 对外提供三种有限值: + +| 值 | 证据与处理 | +| --- | --- | +| `not_dispatched` | 当前执行已失败,未发出远端写入,没有未知提交或仍持有执行权的后台子任务;同时记录 `mutation_started=false`,释放本次占用 | +| `acknowledged` | 对应操作成功完成并保存回执;普通电源命令只确认命令被接纳 | +| `unknown` | 写入或本地提交结果仍可能不明,或后台执行仍待确认;不能据此重放或解锁 | + +外层在调用子应用前仍持久保存 `mutation_started=true`,防止进程崩溃丢失写入边界。 +只有本次存活执行的完整证明才能改为 false。状态/XML/存储检查失败和排队超时 +不再仅因 502/503/504 变成永久锁。协调器确认被另一执行拒绝时传递独立准入证明; +拒绝本次 VM B 的请求只释放 B 的新占用,VM A 原有未决存储占用保持不变。 +单独的错误码、旧数据库字段或先前被拒绝的证据不能覆盖后续实际写入、取消、 +CAS 响应丢失或数据库异常。宿主配置提交及 VPC/Fabric 的 SSH 工作流不使用此例外。 + +异步重装、快照和救援由执行线程在未写入且本地任务/恢复目录处理成功后保存证明; +HTTP 返回 202、队列尚未运行或父请求没有写入都不代表任务已经结束。 +公网 IP 变更须先成功撤销本次新 IP 预留,才能恢复未派发状态;原 IP 保持不变。 +创建退款仍要求独立的虚拟机/卷不存在与归属、凭据、IP 清理证明, +`not_dispatched` 本身不能授权退款。 + +PHP 对精确绑定原操作编号的未派发失败恢复用户删除准入状态;到期删除保持到期策略, +不重新激活实例。旧任务、旧持久占用和旧 RPC 标记不做迁移或自动解除。 +依据:[libvirt 错误 API](https://libvirt.org/html/libvirt-virterror.html) 与 +[Python 绑定说明](https://libvirt.org/python.html);上面的执行证明是面板自己的协议, +不是 libvirt 错误码或 HTTP 状态码提供的事务保证。 + ### 普通电源命令与状态观察(2026-10-13) `POST vms/{uuid}/actions` 的 `start/shutdown/reboot/poweroff/reset` 在 libvirt @@ -78,7 +109,8 @@ desired/applied/usable,也不改变到期/流量断网策略;独立 VPC 协 | `coordination_uncertain` / 503 | 旧执行结果未确认;禁止自动接管 | | `coordination_unavailable` / 503 | 数据库占用/提交结果无法确认;停止后续修改并保留证据 | -后两项与 libvirt 未知写入错误一样保留重装凭据、救援 XML、快照目录和原任务,不进入通用失败补偿。 +后两项缺少上述当前执行的未派发证明时,与 libvirt 未知写入错误一样保留重装凭据、 +救援 XML、快照目录和原任务,不进入通用失败补偿。拒绝新执行不会解除旧阻塞执行。 ## 升级与人工恢复 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 25b10a25..b59dba1a 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,36 @@ # 本版验证记录 +## 2026-10-13:未派发失败不再留下或扩散 VM 持久锁 + +- 状态/XML/创建前检查失败、RPC 排队超时以及被其他 VM 的存储占用拒绝, + 仅在当前执行具备完整无写入证明时保存 `failed`、`mutation_started=false`、 + `execution_outcome=not_dispatched` 并释放本次占用。旧阻塞执行保持原样。 + 写入、取消、数据库/CAS 提交不明和未完成的后台交接继续保留保护。 +- 重装、快照、救援的后台未派发失败须成功处理本地目录与任务记录;公网 IP + 变更须先回滚新 IP 预留。创建退款仍单独核对不存在与清理证明。 + PHP 只对精确绑定的未派发删除失败恢复用户操作准入,不恢复到期删除实例。 +- Python 19 套相关回归统一执行:**738 passed、81 subtests passed、3 failed**。 + 覆盖四个新增 prewrite 测试文件,以及 RPC、libvirt 服务、资源锁、持久协调、 + 后台交接、停机、宿主范围、连接准入、应用并发与签名引擎边界。 + 新增负向用例包括写后失败、伪造/缺少证明、旧锁不变、丢失 CAS 回执、 + 先被拒绝后发生未知结果、后台交接竞态、数据库错误经 HTTP 序列化、 + IP 与快照目录回滚失败、救援 force/exit 恢复原任务归属。 +- 三项 Python 失败均在修改前 `2b87fe89f2fb570be9a3a1f10e952a9fa36efda8` + 的独立源码快照、相同 Python 3.12 依赖中复现: + `test_cloud_template_import_job_and_create_dispatch`、 + `test_reinstall_cleanup_pending_is_not_reported_success`、 + `test_fabric_is_engine_global_and_super_admin_write_only`。 + 本次仅为跳过 RPC 的已有模拟创建/删除服务补上写入标记,未放宽上述失败断言。 +- PHP 8.5 WASM + SQLite/Eloquent:扩展的 `CloudPowerReceiptWorkflowTest.php`、 + 新增 `CloudPrewriteFailureWorkflowTest.php`、`CloudSnapshotWorkflowTest.php`、 + `CloudOperationErrorCatalogTest.php` 均通过。使用真实客户端/worker 验证 + 503 明确失败只退款释放一次、缺少清理证明不退款、后续强制断电、删除准入恢复 + 及到期策略保持。额外执行 `CloudFirewallWorkflowTest.php` 遇到第 517 行既有 + `GuzzleHttp\Handler\HandlerStack` 导入错误;同样在上述修改前源码快照复现。 +- `git diff --check` 通过,独立复核未发现剩余阻断问题。本次无数据库 schema + 变更、历史任务迁移或自动解锁;未部署生产、未执行真实 libvirt/MariaDB 验收。 + 执行协议见 [引擎协调说明](ENGINE_COORDINATION.md#执行前失败与准入拒绝2026-10-13)。 + ## 2026-10-13:普通电源命令回执与状态观察分离 - `start/shutdown/reboot/poweroff/reset` 的 libvirt 调用成功后返回 diff --git a/engine/tests/test_background_prewrite_admission.py b/engine/tests/test_background_prewrite_admission.py new file mode 100644 index 00000000..c2ffa4cd --- /dev/null +++ b/engine/tests/test_background_prewrite_admission.py @@ -0,0 +1,315 @@ +"""Live prewrite receipts settle only the worker that produced them.""" + +import inspect +import json +import threading +import time + +import pytest + +from test_app import ( + VM_ID, configure_host, configure_reinstall_case, configure_snapshot_case, + setup_admin, web, # noqa: F401 +) +from test_durable_jobs import MemoryCAS, jobs # noqa: F401 +from virsh_panel import app as app_module +from virsh_panel.coordination import ( + CoordinationError, DurableCoordinator, execution_context, + current_execution, execution_failure_before_write, mark_admission_rejected, + mark_execution_unknown, mark_remote_write_started, +) +from virsh_panel.database import DatabaseError +from virsh_panel.durable_jobs import read_job_outcome, submit_durable +from virsh_panel.libvirt_service import VMError + + +@pytest.mark.parametrize("error_code", [ + None, "libvirt_execution_timeout", "libvirt_execution_stopped", + "libvirt_connection_failed", "coordination_uncertain", +]) +def test_live_failed_worker_receipt_releases_claim(jobs, error_code): + store, coordinator, parent, executor = jobs + + def worker(): + if error_code == "coordination_uncertain": + mark_admission_rejected() + assert execution_failure_before_write(error_code) + store.set("snapshot_job", "job-one", json.dumps({ + "status": "failed", "error_code": error_code, + "mutation_started": False, "execution_outcome": "not_dispatched", + })) + + with execution_context(coordinator, parent, tracked_mutations=True): + future = submit_durable( + executor, worker, + outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"), + job_kind="snapshot_job", job_id="job-one") + future.result(timeout=2) + assert list(coordinator.snapshot()) == [parent.token] + coordinator.release(parent) + replacement = coordinator.claim(resources=parent.resources) + assert coordinator.release(replacement) + + +@pytest.mark.parametrize("unsafe", ["untracked", "prior_write", "unknown", "old_record", "database"]) +def test_stored_prewrite_flag_does_not_override_missing_or_contrary_evidence(jobs, unsafe): + store, coordinator, parent, executor = jobs + record = { + "status": "failed", "error_code": "libvirt_execution_timeout", + "mutation_started": False, "execution_outcome": "not_dispatched", + } + if unsafe == "unknown": + record["error_code"] = "libvirt_execution_uncertain" + if unsafe == "old_record": + record.pop("execution_outcome") + + def worker(): + if unsafe == "prior_write": + mark_remote_write_started() + if unsafe == "database": + mark_execution_unknown() + store.set("snapshot_job", "job-one", json.dumps(record)) + + with execution_context(coordinator, parent, tracked_mutations=unsafe != "untracked"): + future = submit_durable( + executor, worker, + outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"), + job_kind="snapshot_job", job_id="job-one") + future.result(timeout=2) + child = next(row for row in coordinator.snapshot().values() if row["kind"] == "background") + assert child["state"] == "uncertain" + + +def test_tracked_unwritten_queued_and_running_worker_still_blocks_competitors(jobs): + store, coordinator, parent, executor = jobs + occupied, worker_started = threading.Event(), threading.Event() + release_queue, release_worker = threading.Event(), threading.Event() + + def occupy(): + occupied.set() + assert release_queue.wait(2) + + def worker(): + worker_started.set() + assert execution_failure_before_write("libvirt_execution_timeout") + assert release_worker.wait(2) + store.set("snapshot_job", "job-one", json.dumps({"status": "completed"})) + + blocker = executor.submit(occupy) + assert occupied.wait(2) + try: + with execution_context(coordinator, parent, tracked_mutations=True): + future = submit_durable( + executor, worker, + outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"), + job_kind="snapshot_job", job_id="job-one") + coordinator.release(parent) + with pytest.raises(CoordinationError): + coordinator.claim(resources=parent.resources) + release_queue.set() + blocker.result(timeout=2) + assert worker_started.wait(2) + with pytest.raises(CoordinationError): + coordinator.claim(resources=parent.resources) + finally: + release_queue.set() + release_worker.set() + future.result(timeout=2) + assert coordinator.snapshot() == {} + + +def instrument_background(monkeypatch, *, write=False): + """Supply the live RPC evidence context to the app's fake host worker.""" + coordinator = DurableCoordinator(MemoryCAS()) + parent = coordinator.claim(resources=("vm:host-a:" + VM_ID,)) + original = app_module.submit_durable + seen = {} + + def submit(executor, fn, *args, **kwargs): + kind = kwargs["job_kind"] + seen[kind] = inspect.getclosurevars(fn).nonlocals + + def run(*worker_args, **worker_kwargs): + with execution_context(coordinator, parent, tracked_mutations=True): + seen["execution"] = current_execution() + if write: + mark_remote_write_started() + return fn(*worker_args, **worker_kwargs) + + return original(executor, run, *args, **kwargs) + + monkeypatch.setattr(app_module, "submit_durable", submit) + return seen + + +def wait_job(seen, kind, job_id): + jobs_by_kind = { + "reinstall_job": "reinstall_jobs", "rescue_job": "rescue_jobs", + "snapshot_job": "snapshot_jobs", + } + for _ in range(200): + saved = dict(seen[kind][jobs_by_kind[kind]][job_id]) + if saved["status"] not in {"queued", "running"}: + return saved + time.sleep(0.01) + pytest.fail("background worker did not settle") + + +@pytest.mark.parametrize("write", [False, True]) +def test_reinstall_preflight_timeout_does_not_create_new_pending_guard(web, monkeypatch, write): + client, _state, _token, service, _manager, _ = web + csrf, template_id, _ = configure_reinstall_case(web) + seen = instrument_background(monkeypatch, write=write) + + def fail(*_args, **_kwargs): + raise VMError("preflight read timed out", 503, error_code="libvirt_execution_timeout") + + service.reinstall_cloud_vm = fail + headers = {"X-CSRF-Token": csrf} + response = client.post(f"/api/v1/vms/{VM_ID}/reinstall", json={ + "template_id": template_id, "confirmation_name": "cloud-vm", + }, headers=headers) + assert response.status_code == 202, response.text + result = wait_job(seen, "reinstall_job", response.json()["job_id"]) + assert result["status"] == ("uncertain" if write else "failed") + assert (result.get("execution_outcome") == "not_dispatched") is (not write) + power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers) + assert power.status_code == (409 if write else 200), power.text + + +@pytest.mark.parametrize("write", [False, True]) +def test_snapshot_preflight_timeout_rolls_back_catalog_before_releasing_guard(web, monkeypatch, write): + client, service, csrf = configure_snapshot_case(web) + seen = instrument_background(monkeypatch, write=write) + service.snapshot_create_error = VMError( + "preflight read timed out", 503, error_code="libvirt_execution_timeout") + key = "d" * 32 + headers = {"X-CSRF-Token": csrf} + response = client.post(f"/api/v1/vms/{VM_ID}/snapshots", json={ + "snapshot_key": key, "name": "preflight", "consistency_mode": "crash", + }, headers=headers) + assert response.status_code == 202, response.text + result = wait_job(seen, "snapshot_job", response.json()["job_id"]) + assert result["status"] == ("uncertain" if write else "failed") + assert (result.get("execution_outcome") == "not_dispatched") is (not write) + snapshot = client.get(f"/api/v1/vms/{VM_ID}/snapshots/{key}") + assert snapshot.status_code == (200 if write else 404), snapshot.text + power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers) + assert power.status_code == (409 if write else 200), power.text + + +def configure_rescue(web): + client, _state, token, service, _manager, _ = web + csrf = setup_admin(client, token) + configure_host(client, csrf) + headers = {"X-CSRF-Token": csrf} + iso = bytearray(1024 * 1024) + for sector, kind in ((16, 1), (17, 0), (18, 255)): + offset = sector * 2048 + iso[offset] = kind + iso[offset + 1:offset + 6] = b"CD001" + iso[offset + 6] = 1 + iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION" + iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little") + response = client.post("/api/v1/rescue-templates?name=Rescue", content=bytes(iso), headers=headers) + assert response.status_code == 202, response.text + for _ in range(200): + imported = client.get( + f"/api/v1/rescue-templates/import-jobs/{response.json()['job_id']}").json() + if imported["status"] not in {"queued", "running"}: + break + time.sleep(0.01) + assert imported["status"] == "completed" + template = client.get("/api/v1/rescue-templates").json()["templates"][0] + return client, service, headers, template + + +@pytest.mark.parametrize("write", [False, True]) +def test_rescue_preflight_timeout_clears_only_unstarted_entry(web, monkeypatch, write): + client, service, headers, template = configure_rescue(web) + seen = instrument_background(monkeypatch, write=write) + service.rescue_enter_error = VMError( + "preflight read timed out", 503, error_code="libvirt_execution_timeout") + response = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={ + "template_id": template["id"], "confirmation_name": "alma-test", + }, headers=headers) + assert response.status_code == 202, response.text + result = wait_job(seen, "rescue_job", response.json()["job_id"]) + assert result["status"] == ("uncertain" if write else "failed") + assert (result.get("execution_outcome") == "not_dispatched") is (not write) + assert sum(call[0] == "prepare_rescue_vm" for call in service.calls) == 1 + power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers) + assert power.status_code == (409 if write else 200), power.text + + +@pytest.mark.parametrize("kind", ["reinstall", "snapshot", "rescue"]) +def test_database_failure_never_produces_worker_no_dispatch_receipt(web, monkeypatch, kind): + error = DatabaseError("lost database commit response") + if kind == "reinstall": + client, _state, _token, service, _manager, _ = web + csrf, template_id, _ = configure_reinstall_case(web) + headers = {"X-CSRF-Token": csrf} + body = {"template_id": template_id, "confirmation_name": "cloud-vm"} + resource = "reinstall" + + def fail(*_args, **_kwargs): + raise error + + service.reinstall_cloud_vm = fail + elif kind == "snapshot": + client, service, csrf = configure_snapshot_case(web) + headers = {"X-CSRF-Token": csrf} + body = {"snapshot_key": "a" * 32, "name": "test", "consistency_mode": "crash"} + resource = "snapshots" + service.snapshot_create_error = error + else: + client, service, headers, template = configure_rescue(web) + body = {"template_id": template["id"], "confirmation_name": "alma-test"} + resource = "rescue" + service.rescue_enter_error = error + seen = instrument_background(monkeypatch) + response = client.post(f"/api/v1/vms/{VM_ID}/{resource}", json=body, headers=headers) + assert response.status_code == 202, response.text + result = wait_job(seen, kind + "_job", response.json()["job_id"]) + assert result.get("execution_outcome") != "not_dispatched" + assert result.get("mutation_started") is not False + assert seen["execution"].execution_unknown.is_set() + + +@pytest.mark.parametrize("action", ["force", "exit"]) +def test_unstarted_rescue_transition_restores_previous_recovery_row(web, monkeypatch, action): + client, service, headers, template = configure_rescue(web) + service.rescue_pending_manual_restart = action == "force" + entered = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={ + "template_id": template["id"], "confirmation_name": "alma-test", + }, headers=headers) + assert entered.status_code == 202, entered.text + for _ in range(200): + previous = client.get(f"/api/v1/vms/{VM_ID}/rescue").json() + if previous["status"] in {"active", "staged"}: + break + time.sleep(0.01) + assert previous["status"] == ("staged" if action == "force" else "active") + seen = instrument_background(monkeypatch) + + def fail(*_args, **_kwargs): + raise VMError("read timeout", 503, error_code="libvirt_execution_timeout") + + monkeypatch.setattr(service, "force_start_staged_rescue_vm" if action == "force" + else "exit_rescue_vm", fail) + resource = "force-start" if action == "force" else "exit" + response = client.post(f"/api/v1/vms/{VM_ID}/rescue/{resource}", json={ + "confirmation_name": "alma-test", + **({"ack_data_loss": True} if action == "force" else {}), + }, headers=headers) + assert response.status_code == 202, response.text + result = wait_job(seen, "rescue_job", response.json()["job_id"]) + assert result["status"] == "failed" + assert result["execution_outcome"] == "not_dispatched" + restored = client.get(f"/api/v1/vms/{VM_ID}/rescue").json() + assert restored["status"] == previous["status"] + # The new failed transition must not replace the original recovery owner. + namespace = seen["rescue_job"] + profile = web[4].current() + saved = namespace["load_rescue_state"](profile, VM_ID) + assert saved["job_id"] == entered.json()["job_id"] diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 1a388d80..0bfdf1ba 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -24,6 +24,7 @@ from virsh_panel.app import create_app from virsh_panel.cloud_templates import (TemplateError, template_id_for, template_scope, template_volume_name) from virsh_panel.config import Settings +from virsh_panel.coordination import mark_remote_write_started from virsh_panel.database import DatabaseError, DatabaseTemplateStore from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore, LogStore, MainStore, TRUSTED_ACTOR) @@ -206,12 +207,19 @@ class Service(FakeVMService): def create_vm(self, spec): self.calls.append(("create_vm", spec)) + # This adapter skips the RPC transport; model its write boundary so + # post-dispatch failures remain distinct from no-write preflights. + mark_remote_write_started() if self.create_failure: from virsh_panel.libvirt_service import VMError raise VMError("宿主机失联,回滚状态不明", 502) self.vm.update(uuid=spec["vm_uuid"], name=spec["name"]) return self.vm.copy() + def delete_vm(self, vm_id, delete_disks): + mark_remote_write_started() + return super().delete_vm(vm_id, delete_disks) + def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False, legacy_name_collision=False): self.absent_checks.append( diff --git a/engine/tests/test_host_service_scope.py b/engine/tests/test_host_service_scope.py index e5c825cf..a291d79a 100644 --- a/engine/tests/test_host_service_scope.py +++ b/engine/tests/test_host_service_scope.py @@ -65,7 +65,7 @@ def host_app(tmp_path, monkeypatch): monkeypatch.setattr(MetricsCollector, "start", lambda _self: None) @contextmanager - def resource_guard(resources, *, timeout, connection_failure_safe): + def resource_guard(resources, *, timeout, connection_failure_safe, tracked_mutations): assert connection_failure_safe is True locks.append(tuple(resources)) yield diff --git a/engine/tests/test_network_prewrite_admission.py b/engine/tests/test_network_prewrite_admission.py new file mode 100644 index 00000000..f7a86717 --- /dev/null +++ b/engine/tests/test_network_prewrite_admission.py @@ -0,0 +1,136 @@ +"""A failed public-IP preflight releases only a confirmed local reservation.""" + +import json + +import pytest + +from test_engine_boundary import USER, VM_ID, boundary, call # noqa: F401 +from virsh_panel.coordination import mark_remote_write_started +from virsh_panel.database import DatabaseError +from virsh_panel.libvirt_service import VMError +from virsh_panel.metrics import host_scope + + +def test_database_error_handler_cannot_turn_serialized_503_into_no_dispatch(boundary, monkeypatch): + client, engine, store, _logs, services = boundary + assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 + components = engine.registry.components["hk"] + components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, USER["user_id"]) + + def failed_action(*_args, **_kwargs): + raise DatabaseError("lost database commit response") + + monkeypatch.setattr(services["hk"], "action", failed_action) + operation_id = "database-error-http-boundary" + response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", { + **USER, "operation_id": operation_id, + }, {"action": "poweroff"}) + assert response.status_code == 503, response.text + record = json.loads(store.get("operation", operation_id)) + assert record["status"] == "uncertain", record + assert record.get("execution_outcome") != "not_dispatched" + assert engine.coordinator.operation_claim(operation_id) is not None + + +def test_failed_snapshot_admission_receipt_does_not_hide_catalog_rollback_failure(boundary, monkeypatch): + client, engine, store, _logs, services = boundary + assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 + components = engine.registry.components["hk"] + components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, USER["user_id"]) + services["hk"].vm.update(managed=True, os_source="cloud_template", + rescue_mode=False, rescue_staged=False) + original_set, original_delete = components.store.set, components.store.delete + + def failed_job_save(kind, key, value): + if kind == "snapshot_job": + raise DatabaseError("initial job save failed") + return original_set(kind, key, value) + + def failed_catalog_rollback(kind, key): + if kind == "vm_snapshot": + raise DatabaseError("snapshot catalog rollback failed") + return original_delete(kind, key) + + monkeypatch.setattr(components.store, "set", failed_job_save) + monkeypatch.setattr(components.store, "delete", failed_catalog_rollback) + operation_id = "snapshot-local-rollback-unknown" + response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots", { + **USER, "operation_id": operation_id, + }, {"snapshot_key": "e" * 32, "name": "rollback failure", "consistency_mode": "crash"}) + assert response.status_code == 503, response.text + record = json.loads(store.get("operation", operation_id)) + assert record["status"] == "uncertain", record + assert record.get("execution_outcome") != "not_dispatched" + assert engine.coordinator.operation_claim(operation_id) is not None + assert components.store.get("vm_snapshot", f"{VM_ID}:" + "e" * 32) is not None + assert not any(row[0] == "create_live_snapshot" for row in services["hk"].calls) + + +@pytest.mark.parametrize("failure", ["read", "write", "rollback", "reservation"]) +def test_network_prewrite_failure_requires_successful_ip_rollback(boundary, monkeypatch, failure): + client, engine, store, _logs, services = boundary + assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 + components = engine.registry.components["hk"] + profile = components.manager.profile + profile.update({ + "host": "8.8.8.3", "default_network": "bridge:br0", + "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", + "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], + "public_vm_prefix": 28, + }) + service = services["hk"] + service.vm.update(managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28") + scope = host_scope(profile) + components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) + original_claim = json.dumps({"vm_uuid": VM_ID}) + components.store.set("ip_claim", f"{scope}.8.8.8.4", original_claim) + expected_key = f"{scope}.8.8.8.6" + calls = [] + + def fail_service(*_args, **_kwargs): + calls.append("service") + if failure == "write": + mark_remote_write_started() + raise VMError("preflight read failed", 502) + + monkeypatch.setattr(service, "change_cloud_vm_network", fail_service) + if failure == "rollback": + original_delete = components.store.delete + + def failed_delete(kind, key): + if kind == "ip_claim" and key == expected_key: + raise DatabaseError("reservation rollback failed") + return original_delete(kind, key) + + monkeypatch.setattr(components.store, "delete", failed_delete) + if failure == "reservation": + original_insert = components.store.insert + + def lost_insert(kind, key, value): + result = original_insert(kind, key, value) + if kind == "ip_claim" and key == expected_key: + raise DatabaseError("reservation commit response lost") + return result + + monkeypatch.setattr(components.store, "insert", lost_insert) + + operation_id = "network-prewrite-" + failure + response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", { + **USER, "operation_id": operation_id, + }, {"public_ip_cidr": "8.8.8.6/28", "confirm_restart": True}) + assert response.status_code >= 500, response.text + record = json.loads(store.get("operation", operation_id)) + assert components.store.get("ip_claim", f"{scope}.8.8.8.4") == original_claim + assert service.vm["public_ip_cidr"] == "8.8.8.4/28" + if failure == "read": + assert record["status"] == "failed", record + assert record["network_mutation_started"] is False + assert record["execution_outcome"] == "not_dispatched" + assert components.store.get("ip_claim", expected_key) is None + assert engine.coordinator.operation_claim(operation_id) is None + else: + assert record["status"] == "uncertain", record + assert record.get("execution_outcome") != "not_dispatched" + assert components.store.get("ip_claim", expected_key) is not None + assert engine.coordinator.operation_claim(operation_id) is not None + assert calls == ([] if failure == "reservation" else ["service"]) diff --git a/engine/tests/test_prewrite_admission.py b/engine/tests/test_prewrite_admission.py new file mode 100644 index 00000000..97aa2596 --- /dev/null +++ b/engine/tests/test_prewrite_admission.py @@ -0,0 +1,193 @@ +"""No-dispatch failures release only this live VM execution's claims.""" + +from types import SimpleNamespace + +import pytest + +from test_connection_failure_admission import MemoryStore, VM_ID, VMError, VMService +from virsh_panel import coordination, libvirt_service as service_module +from virsh_panel.coordination import ( + CoordinationClaimRejected, CoordinationError, DurableCoordinator, + execution_context, execution_failure_before_write, mark_admission_rejected, + mark_remote_write_started, resource_guard, +) +from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime, RPCSession + + +@pytest.fixture +def tracked_service(monkeypatch): + coordinator = DurableCoordinator(MemoryStore()) + connection = SimpleNamespace() + session = SimpleNamespace(connection=connection, failure=None, close=lambda: None) + runtime = SimpleNamespace(open=lambda *_args, **_kwargs: session, queue_timeout=0.01) + monkeypatch.setattr(service_module, "get_runtime", lambda: runtime) + service = VMService("test", "images", "iso", "default", execution_scope="7") + return coordinator, service, connection, runtime + + +@pytest.mark.parametrize("stage", ["state", "clean-traffic", "queue"]) +def test_failed_vm_preflight_releases_vm_storage_and_allocation(tracked_service, stage): + coordinator, service, connection, runtime = tracked_service + request = coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + writes = [] + + def read_failed(): + raise service_module.libvirt.libvirtError("read denied before any mutation") + + connection.lookupByUUIDString = lambda _uuid: SimpleNamespace( + isActive=read_failed, create=lambda: writes.append("create")) + if stage == "queue": + def queue_full(*_args, **_kwargs): + raise ExecutionFailure("libvirt_execution_timeout") + runtime.open = queue_full + with execution_context(coordinator, request, tracked_mutations=True): + with pytest.raises(VMError) as raised: + if stage == "clean-traffic": + with service._operation_guard(VM_ID, storage=True, allocation=True): + service._require_clean_traffic_filter(connection) + else: + service.action(VM_ID, "start") + assert execution_failure_before_write(raised.value.error_code) + assert set(coordinator.snapshot()) == {request.token} + assert writes == [] + assert coordinator.release(request) + next_request = coordinator.claim(resources=("storage:7", "allocation:7", f"vm:7:{VM_ID}")) + assert coordinator.release(next_request) + + +def test_other_vm_storage_fence_does_not_contaminate_rejected_vm(tracked_service): + coordinator, service, _connection, _runtime = tracked_service + blocker = coordinator.claim(resources=("storage:7", "vm:7:other")) + coordinator.retain(blocker) + before = coordinator.snapshot()[blocker.token] + request = coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + with execution_context(coordinator, request, tracked_mutations=True): + with pytest.raises(VMError) as raised: + with service._operation_guard(VM_ID, storage=True): + pytest.fail("No host work may begin after conflicting admission") + assert raised.value.error_code == "coordination_uncertain" + assert execution_failure_before_write(raised.value.error_code) + assert not coordination._uncertain_exception(raised.value) + assert coordinator.release(request) + assert coordinator.snapshot() == {blocker.token: before} + # VM B's later power action needs only its own VM resource. + next_request = coordinator.claim() + with execution_context(coordinator, next_request, tracked_mutations=True): + with service._operation_guard(VM_ID): + pass + assert coordinator.release(next_request) + + +@pytest.mark.parametrize("untracked", ["native", "mixed"]) +def test_untracked_execution_cannot_infer_no_write(tracked_service, untracked): + coordinator, service, _connection, _runtime = tracked_service + if untracked == "native": + service.execution_scope = None + request = coordinator.claim() + with execution_context(coordinator, request): + with pytest.raises(VMError): + with service._operation_guard(VM_ID): + raise VMError("untracked failure", 502) + assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values()) + + +@pytest.mark.parametrize("prior", ["write", "unknown", "cancel"]) +def test_no_write_marker_never_releases_actual_or_unknown_execution(tracked_service, prior): + coordinator, service, _connection, _runtime = tracked_service + request = coordinator.claim() + with execution_context(coordinator, request, tracked_mutations=True): + error = VMError("later read failed", 502) + if prior == "write": + mark_remote_write_started() + elif prior == "unknown": + error = VMError("unknown receipt", 502, error_code="libvirt_execution_uncertain") + else: + import asyncio + error = asyncio.CancelledError() + with pytest.raises(type(error)): + with service._operation_guard(VM_ID): + raise error + assert not execution_failure_before_write(getattr(error, "error_code", None)) + assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values()) + + +def test_error_code_alone_cannot_claim_definite_admission_rejection(tracked_service): + coordinator, service, _connection, _runtime = tracked_service + request = coordinator.claim() + with execution_context(coordinator, request, tracked_mutations=True): + with pytest.raises(VMError): + with service._operation_guard(VM_ID): + raise VMError("unproven result", 503, error_code="coordination_uncertain") + assert not execution_failure_before_write("coordination_uncertain") + assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values()) + + +def test_lost_claim_commit_invalidates_earlier_admission_proof(tracked_service, monkeypatch): + coordinator, _service, _connection, _runtime = tracked_service + request = coordinator.claim() + compare = coordinator.store.compare_and_set + + def lose_reply(*args): + assert compare(*args) + raise OSError("commit succeeded but reply lost") + + with execution_context(coordinator, request, tracked_mutations=True): + mark_admission_rejected() + assert execution_failure_before_write("coordination_uncertain") + monkeypatch.setattr(coordinator.store, "compare_and_set", lose_reply) + with pytest.raises(CoordinationError) as raised: + with resource_guard((f"vm:7:{VM_ID}",)): + pytest.fail("Unknown durable admission must not dispatch") + assert raised.value.error_code == "coordination_unavailable" + assert not execution_failure_before_write("coordination_uncertain") + assert not execution_failure_before_write(None) + assert coordination._uncertain_exception(VMError( + "later queue timeout", 504, error_code="libvirt_execution_timeout")) + assert len(coordinator.snapshot()) == 2 + + +def test_own_uncertain_claim_cannot_reuse_another_claims_rejection_receipt(): + coordinator = DurableCoordinator(MemoryStore()) + request = coordinator.claim() + with execution_context(coordinator, request, tracked_mutations=True): + mark_admission_rejected() + coordinator.retain(request) + with pytest.raises(CoordinationError) as raised: + coordinator.add_resources(request, ("storage:7",)) + assert not isinstance(raised.value, CoordinationClaimRejected) + assert not execution_failure_before_write(raised.value.error_code) + assert not execution_failure_before_write(None) + + +def test_confirmed_rpc_fence_rejection_keeps_original_fence_only(tmp_path): + coordinator = DurableCoordinator(MemoryStore()) + request = coordinator.claim() + runtime = LibvirtRuntime(tmp_path / "rpc") + fence = runtime._directory("7") / ("a" * 32 + ".json") + fence.write_text("previous unknown write") + session = RPCSession(runtime, "test", "7", 5) + with execution_context(coordinator, request, tracked_mutations=True): + with pytest.raises(ExecutionFailure) as raised: + session.call(1, "destroy", (), {}) + assert raised.value.error_code == "libvirt_execution_blocked" + assert execution_failure_before_write(raised.value.error_code) + assert session.intent is None + assert list(runtime.state_dir.rglob("*.json")) == [fence] + # A genuine subsequent unknown error cannot reuse that admission proof. + other = RPCSession(runtime, "test", "7", 5) + other._fail("libvirt_execution_uncertain", outcome_unknown=True) + assert not execution_failure_before_write("libvirt_execution_blocked") + assert fence.read_text() == "previous unknown write" + + +def test_claim_rejection_in_mixed_workflow_does_not_erase_unknown_ssh_effects(): + coordinator = DurableCoordinator(MemoryStore()) + blocker = coordinator.claim(resources=("storage:7",)) + coordinator.retain(blocker) + request = coordinator.claim() + with execution_context(coordinator, request): + with pytest.raises(CoordinationClaimRejected) as raised: + with resource_guard(("storage:7",)): + pytest.fail("Conflicting claim admitted") + assert coordination._uncertain_exception(raised.value) + assert not execution_failure_before_write(raised.value.error_code) diff --git a/engine/tests/test_prewrite_execution_boundary.py b/engine/tests/test_prewrite_execution_boundary.py new file mode 100644 index 00000000..8494c860 --- /dev/null +++ b/engine/tests/test_prewrite_execution_boundary.py @@ -0,0 +1,175 @@ +"""Signed operation receipts distinguish no dispatch from unknown execution.""" + +import asyncio +import json + +import pytest + +from test_engine_boundary import ( + CREATE_BODY, USER, VM_ID, boundary, call, host_scope, +) +from virsh_panel.coordination import ( + execution_context, mark_execution_unknown, mark_remote_write_started, +) +from virsh_panel.libvirt_service import VMError, VMService + + +def prepare(boundary): + client, engine, store, _, services = boundary + call(client, "GET", "/internal/v1/hosts/hk/vms") + component = engine.registry.components["hk"] + component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) + return client, engine, store, component, services["hk"] + + +@pytest.mark.parametrize("code", [None, "libvirt_execution_timeout", "libvirt_execution_stopped"]) +def test_prewrite_502_finishes_and_allows_followup_power(boundary, monkeypatch, code): + client, engine, store, _, service = prepare(boundary) + path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" + action = service.action + attempts = [] + tracked = VMService("test", "images", "iso", "default", execution_scope="hk") + + def failed(vm_id, command): + attempts.append(command) + with tracked._operation_guard(vm_id): + raise VMError("state/XML read failed before dispatch", 502, error_code=code) + + monkeypatch.setattr(service, "action", failed) + actor = {**USER, "operation_id": "prewrite-power-receipt"} + response = call(client, "POST", path, actor, {"action": "reboot"}) + assert response.status_code == 502, response.text + assert response.json()["status"] == "failed" + assert response.json()["mutation_started"] is False + assert response.json()["execution_outcome"] == "not_dispatched" + assert engine.coordinator.snapshot() == {} + saved = store.get("operation", actor["operation_id"]) + polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}") + assert polled.json()["execution_outcome"] == "not_dispatched" + assert call(client, "POST", path, actor, {"action": "reboot"}).json()["status"] == "failed" + assert store.get("operation", actor["operation_id"]) == saved + assert attempts == ["reboot"] + monkeypatch.setattr(service, "action", action) + forced = call(client, "POST", path, {**USER, "operation_id": "power-after-prewrite"}, + {"action": "poweroff"}) + assert forced.json()["status"] == "succeeded", forced.text + assert forced.json()["execution_outcome"] == "acknowledged" + + +def test_foreign_storage_fence_cannot_spread_to_rejected_vm(boundary, monkeypatch): + client, engine, _, _, service = prepare(boundary) + blocker = engine.coordinator.claim(resources=("storage:hk", "vm:hk:other")) + engine.coordinator.retain(blocker) + original = engine.coordinator.snapshot() + action = service.action + tracked = VMService("test", "images", "iso", "default", execution_scope="hk") + + def rejected(vm_id, _action): + with tracked._operation_guard(vm_id, storage=True): + pytest.fail("A refused request must not execute") + + monkeypatch.setattr(service, "action", rejected) + path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" + response = call(client, "POST", path, {**USER, "operation_id": "storage-rejected-vm"}, + {"action": "start"}) + assert response.status_code == 503, response.text + assert response.json()["status"] == "failed" + assert response.json()["execution_outcome"] == "not_dispatched" + assert engine.coordinator.snapshot() == original + monkeypatch.setattr(service, "action", action) + next_response = call(client, "POST", path, {**USER, "operation_id": "storage-next-power"}, + {"action": "poweroff"}) + assert next_response.status_code == 200, next_response.text + assert engine.coordinator.snapshot() == original + + +@pytest.mark.parametrize("absence", [True, False, "read_failed"]) +def test_create_prewrite_failure_unlocks_but_refund_needs_cleanup(boundary, monkeypatch, absence): + client, engine, _, component, service = prepare(boundary) + tracked = VMService("test", "images", "iso", "default", execution_scope="hk") + + def failed(spec): + with tracked._operation_guard(spec["vm_uuid"], storage=True, allocation=True): + raise VMError("traffic preflight failed", 502) + + def check(*_args, **_kwargs): + if absence == "read_failed": + raise VMError("absence read unavailable", 504, error_code="libvirt_execution_timeout") + return absence + + monkeypatch.setattr(service, "create_vm", failed) + monkeypatch.setattr(service, "provisioning_absent", check) + actor = {**USER, "can_provision": True, "operation_id": "create-prewrite-proof"} + response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY) + assert response.status_code == 502, response.text + assert response.json()["status"] == "failed" + assert response.json()["execution_outcome"] == "not_dispatched" + assert response.json()["safe_to_refund"] is (absence is True) + assert engine.coordinator.snapshot() == {} + + +@pytest.mark.parametrize("mode", ["write", "unproven_unknown", "database", "unknown_then_409"]) +def test_actual_or_unproven_failure_keeps_vm_protected(boundary, monkeypatch, mode): + client, engine, _, _, service = prepare(boundary) + + def failed(*_args): + if mode == "write": + mark_remote_write_started() + raise VMError("response lost after write", 502) + if mode == "database": + raise OSError("commit reply lost") + if mode == "unknown_then_409": + mark_execution_unknown() + raise VMError("later validation rejection cannot erase unknown commit", 409) + raise VMError("unproven admission outcome", 503, error_code="coordination_uncertain") + + monkeypatch.setattr(service, "action", failed) + path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" + response = call(client, "POST", path, {**USER, "operation_id": "unknown-power-receipt"}, + {"action": "reset"}) + assert response.status_code >= 500 or mode == "unknown_then_409" + claims = engine.coordinator.snapshot() + assert any(row["state"] == "uncertain" and f"vm:hk:{VM_ID}" in row["resources"] + for row in claims.values()) + blocked = call(client, "POST", path, {**USER, "operation_id": "power-still-protected"}, + {"action": "poweroff"}) + assert blocked.status_code == 503 + + +@pytest.mark.parametrize("state", ["pending_child", "write_during_snapshot"]) +def test_no_dispatch_proof_cannot_race_background_handoff(boundary, monkeypatch, state): + _, engine, _, _, _ = prepare(boundary) + parent = engine.coordinator.claim() + with execution_context(engine.coordinator, parent, tracked_mutations=True): + if state == "pending_child": + engine.coordinator.claim(execution_id=parent.execution_id, kind="background") + else: + snapshot = engine.coordinator.snapshot + + def child_finished(): + mark_remote_write_started() + return snapshot() + + monkeypatch.setattr(engine.coordinator, "snapshot", child_finished) + assert not asyncio.run(engine._failed_before_dispatch({}, "libvirt_execution_timeout")) + + +@pytest.mark.parametrize("proof", [True, False]) +def test_background_operation_uses_only_explicit_new_no_dispatch_receipt(boundary, proof): + client, engine, store, component, _ = prepare(boundary) + op_id, job_id = "prewrite-background", "job-prewrite" + job = {"operation_id": op_id, "owner": "php:41", "vm_id": VM_ID, + "status": "failed", "error_code": "coordination_uncertain", "error": "refused"} + if proof: + job.update(mutation_started=False, execution_outcome="not_dispatched") + component.store.set("reinstall_job", job_id, json.dumps(job)) + store.set("operation", op_id, json.dumps({ + "operation_id": op_id, "host_id": "hk", "user_id": 41, "kind": "mutation", + "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "job_id": job_id, + "status": "pending", "mutation_started": True, "execution_id": "a" * 64, + "result": None, "safe_to_refund": False, + })) + result = call(client, "GET", f"/internal/v1/operations/{op_id}").json() + assert result["status"] == ("failed" if proof else "uncertain"), result + assert result["execution_outcome"] == ("not_dispatched" if proof else "unknown") + assert result["mutation_started"] is (not proof) diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index 1623cf41..58d17fb9 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -74,7 +74,10 @@ from .shutdown import ( ManagedThreadPoolExecutor, close_executors_async, run_bounded_close, shutdown_scope, stop_executors, ) -from .coordination import CoordinationError, current_execution, resource_guard +from .coordination import ( + CoordinationError, current_execution, execution_failure_before_write, + mark_execution_unknown, resource_guard, +) from .durable_jobs import mark_not_started, read_job_outcome, submit_durable from .resource_locks import ResourceLockRegistry, ResourceLockTimeout from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater @@ -2193,6 +2196,28 @@ def create_app( @app.exception_handler(DatabaseError) async def handle_database_error(_request: Request, exc: DatabaseError): + # A lost database commit can be ambiguous without any libvirt write. + # The sole exception is a durable pre-executor rejection whose local + # rollback completed, without a child that may still perform work. + execution = current_execution() + not_started = False + if execution is not None and execution_failure_before_write(None): + try: + claims = await asyncio.to_thread(execution.coordinator.snapshot) + owned = [row for row in claims.values() + if row["execution_id"] == execution.claim.execution_id] + not_started = ( + any(row["kind"] == "request" + and row["metadata"].get("failed_before_start") is True + for row in owned) + and not any(row["kind"] == "background" or row["state"] == "uncertain" + for row in owned) + and execution_failure_before_write(None) + ) + except Exception: + pass + if not not_started: + mark_execution_unknown() return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(UpdateError) @@ -4575,10 +4600,17 @@ def create_app( "vm snapshot %s succeeded: job=%s uuid=%s snapshot=%s", action, job_id, vm_id, snapshot_key) except Exception as exc: + if not isinstance(exc, (VMError, CoordinationError, ValueError)): + mark_execution_unknown() + not_dispatched = (not physical_committed + and isinstance(exc, (VMError, CoordinationError)) + and execution_failure_before_write( + getattr(exc, "error_code", None))) uncertain = (physical_committed or isinstance(exc, VMSnapshotUncertainError) - or getattr(exc, "error_code", "") in JOB_UNCERTAIN_CODES) - if not isinstance(exc, (VMError, DatabaseError, ValueError)): + or (not not_dispatched + and getattr(exc, "error_code", "") in JOB_UNCERTAIN_CODES)) + if not not_dispatched and not isinstance(exc, (VMError, DatabaseError, ValueError)): uncertain = True LOG.exception( "VM snapshot %s failed unexpectedly: job=%s uuid=%s", @@ -4601,6 +4633,7 @@ def create_app( save_snapshot_record(record) except Exception: uncertain = True + mark_execution_unknown() LOG.exception("Could not reconcile snapshot record after job failure: %s", job_id) with snapshot_lock: current = snapshot_jobs[job_id] @@ -4613,9 +4646,16 @@ def create_app( ) if error_code is not None: current["error_code"] = error_code + if not_dispatched and not uncertain: + current.update(mutation_started=False, + execution_outcome="not_dispatched") try: save_job("snapshot_job", job_id, current) except DatabaseError: + mark_execution_unknown() + current["status"] = "uncertain" + current.pop("mutation_started", None) + current.pop("execution_outcome", None) LOG.exception("Could not persist snapshot job failure: %s", job_id) # VMService has already closed/reaped the original RPC session. # Do not wait for restart or polling while guest filesystems may @@ -4963,7 +5003,11 @@ def create_app( vm_id=canonical_id, snapshot_key=body.snapshot_key, action="create", consistency=body.consistency_mode) except Exception: - delete_snapshot_record(canonical_id, body.snapshot_key) + try: + delete_snapshot_record(canonical_id, body.snapshot_key) + except Exception: + mark_execution_unknown() + raise raise return queued @@ -5004,7 +5048,11 @@ def create_app( action="restore", consistency=record["consistency"]) except Exception: record["state"] = "ready" - save_snapshot_record(record) + try: + save_snapshot_record(record) + except Exception: + mark_execution_unknown() + raise raise return queued @@ -5041,7 +5089,11 @@ def create_app( action="delete", consistency=record["consistency"]) except Exception: record["state"] = "ready" - save_snapshot_record(record) + try: + save_snapshot_record(record) + except Exception: + mark_execution_unknown() + raise raise return queued @@ -5553,6 +5605,8 @@ def create_app( "awaits cleanup reconciliation" if partial else "succeeded", job_id, vm_id) except Exception as exc: + if not isinstance(exc, (VMError, CoordinationError, TemplateError, ValueError)): + mark_execution_unknown() if isinstance(exc, (VMError, TemplateError, ValueError)): error = str(exc) else: @@ -5561,7 +5615,12 @@ def create_app( with reinstall_lock: job = reinstall_jobs[job_id] committed = vm_reinstalled or job.get("reinstall_committed") is True - interrupted = getattr(exc, "error_code", "") in JOB_UNCERTAIN_CODES + not_dispatched = (not committed + and isinstance(exc, (VMError, CoordinationError)) + and execution_failure_before_write( + getattr(exc, "error_code", None))) + interrupted = (not not_dispatched + and getattr(exc, "error_code", "") in JOB_UNCERTAIN_CODES) if interrupted: # A timed-out read may follow earlier writes in this job. # Keep its credentials and durable progress, but do not @@ -5591,9 +5650,15 @@ def create_app( job["phase"] = "重装失败" job["error"] = error job.pop("generated_password", None) + if not_dispatched: + job.update(mutation_started=False, + execution_outcome="not_dispatched") + if getattr(exc, "error_code", None): + job["error_code"] = exc.error_code try: save_job("reinstall_job", job_id, job) except Exception: + mark_execution_unknown() LOG.exception("Could not persist reinstall status: job=%s", job_id) # A failed terminal write leaves the durable row at its # previous phase. Keep the in-memory guard fail-closed @@ -5601,6 +5666,8 @@ def create_app( # uncertain state; a later process restart will normalize # the old running row the same way. job["status"] = "uncertain" + job.pop("mutation_started", None) + job.pop("execution_outcome", None) job["phase"] = ("重装结果待核对" if committed else "远程执行结果待核对") job.pop("finished_at", None) @@ -5793,10 +5860,13 @@ def create_app( def run_rescue_job(job_id: str, profile: dict, vm_id: str, operation: Literal["enter", "exit", "force"], - network_suspended: bool = False) -> None: + network_suspended: bool = False, + previous_rescue_state: dict | None = None) -> None: vm_scope = host_scope(profile) key = (vm_scope, vm_id) completed = False + before_execution = (dict(previous_rescue_state) + if previous_rescue_state is not None else None) try: with rescue_lock: job = rescue_jobs[job_id] @@ -5810,6 +5880,8 @@ def create_app( state_record = load_rescue_state(profile, vm_id) if state_record is None: raise VMError("救援恢复记录丢失,已停止操作", 409) + if operation == "enter": + before_execution = dict(state_record) # The state row is the durable XML ownership marker consulted # by the VPC lifecycle gate. Older rows may predate the # operation identity; bind them to the authenticated rescue @@ -5854,6 +5926,45 @@ def create_app( save_job("rescue_job", job_id, job) LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id) except Exception as exc: + if not isinstance(exc, (VMError, CoordinationError, TemplateError, ValueError)): + mark_execution_unknown() + if (not completed and isinstance(exc, (VMError, CoordinationError)) + and execution_failure_before_write( + getattr(exc, "error_code", None))): + # No host request was dispatched. Restore only this worker's + # local admission state; a failed readback must not create a + # new recovery barrier for an unchanged guest. + with rescue_lock: + job = rescue_jobs[job_id] + try: + if before_execution is None: + raise DatabaseError("无法确认救援任务开始前的恢复记录") + if operation == "enter": + delete_rescue_state(profile, vm_id) + else: + save_rescue_state(profile, vm_id, before_execution) + job.update( + status="failed", phase="救援操作未执行", + error=str(exc), finished_at=int(time.time()), + mutation_started=False, execution_outcome="not_dispatched", + ) + if getattr(exc, "error_code", None): + job["error_code"] = exc.error_code + save_job("rescue_job", job_id, job) + except Exception: + mark_execution_unknown() + job.update( + status="uncertain", phase="救援登记状态待核对", + error="救援操作尚未派发,但恢复记录清理未能确认", + ) + for field in ("mutation_started", "execution_outcome", "finished_at"): + job.pop(field, None) + LOG.exception("Could not settle unstarted rescue job: job=%s", job_id) + try: + save_job("rescue_job", job_id, job) + except Exception: + LOG.exception("Could not preserve rescue admission failure: job=%s", job_id) + return if getattr(exc, "error_code", "") in JOB_UNCERTAIN_CODES: # A lost RPC response cannot prove that rescue XML or power # state was left unchanged. Retain the original XML and guard; @@ -6068,6 +6179,7 @@ def create_app( previous_status = record["status"] previous_job_id = record.get("job_id") previous_operation_id = record.get("operation_id") + previous_record = dict(record) try: save_job("rescue_job", job_id, rescue_jobs[job_id]) record["status"] = "queued" @@ -6077,6 +6189,7 @@ def create_app( pending_rescues.add(key) submit_durable( rescue_executor, run_rescue_job, job_id, profile, canonical_id, "force", + previous_rescue_state=previous_record, outcome=lambda: read_job_outcome(data_store, "rescue_job", job_id), job_kind="rescue_job", job_id=job_id) except Exception: @@ -6129,6 +6242,7 @@ def create_app( "vm_id": canonical_id, "operation": "exit", "status": "queued", "phase": "等待退出救援模式", } + previous_record = dict(record) save_job("rescue_job", job_id, rescue_jobs[job_id]) record["status"] = "exiting" record["job_id"] = job_id @@ -6139,6 +6253,7 @@ def create_app( submit_durable( rescue_executor, run_rescue_job, job_id, profile, canonical_id, "exit", body.network_suspended, + previous_rescue_state=previous_record, outcome=lambda: read_job_outcome(data_store, "rescue_job", job_id), job_kind="rescue_job", job_id=job_id) except (RuntimeError, CoordinationError) as exc: @@ -6400,10 +6515,23 @@ def create_app( "virt_cloud_mark_network_mutation_started") if callable(mark_network_mutation_started): mark_network_mutation_started() - claim_scheduled_public_ip( - profile, target_pool, - validated["public_ip_cidr"], normalized_id, - ) + mark_network_not_started = request.scope.get( + "virt_cloud_mark_network_mutation_not_started") + try: + claim_scheduled_public_ip( + profile, target_pool, + validated["public_ip_cidr"], normalized_id, + ) + except Exception as exc: + # A database failure may have reserved the new address even + # though libvirt was never called. Keep that outcome fenced. + if (isinstance(exc, VMError) and exc.status < 500 + and execution_failure_before_write(exc.error_code)): + if callable(mark_network_not_started): + mark_network_not_started() + else: + mark_execution_unknown() + raise claim_reserved = True try: network = { @@ -6419,10 +6547,21 @@ def create_app( # A successful libvirt request can lose its response. Never # release a new claim on a 5xx/unknown result; doing so could # allocate the same address to a second guest. - if (claim_reserved and isinstance(exc, VMError) - and exc.status < 500): - release_public_ip_claim( - profile, validated["public_ip_cidr"], normalized_id) + if not isinstance(exc, (VMError, CoordinationError)): + mark_execution_unknown() + not_dispatched = (isinstance(exc, (VMError, CoordinationError)) + and execution_failure_before_write( + getattr(exc, "error_code", None))) + if claim_reserved and (not_dispatched or ( + isinstance(exc, VMError) and exc.status < 500)): + try: + release_public_ip_claim( + profile, validated["public_ip_cidr"], normalized_id) + if not_dispatched and callable(mark_network_not_started): + mark_network_not_started() + except Exception: + mark_execution_unknown() + raise raise try: release_public_ip_claim(profile, old_ip, normalized_id) diff --git a/engine/virsh_panel/coordination.py b/engine/virsh_panel/coordination.py index 6baf306a..8a953702 100644 --- a/engine/virsh_panel/coordination.py +++ b/engine/virsh_panel/coordination.py @@ -46,14 +46,17 @@ class CoordinationError(Exception): class CoordinationAdmissionRejected(CoordinationError): - """A confirmed conflict before a VPC entry can dispatch host work. + """A confirmed admission conflict before this claim can dispatch work. - Only the VPC pre-dataplane lock boundary may produce this exception. The old blocking claim remains authoritative; this exception only proves that the rejected execution did not start its own remote mutation. """ +class CoordinationClaimRejected(CoordinationAdmissionRejected): + """A conflicting claim was observed without committing a new claim.""" + + @dataclass(frozen=True) class Claim: token: str @@ -74,6 +77,13 @@ class ExecutionContext: remote_write_started: threading.Event = field(default_factory=threading.Event, compare=False, repr=False) connection_failure_safe: bool = False + # Opt in only where every remote mutation passes the RPC write hook. + # In particular, this is not implied by read-only SSH connection probes. + tracked_mutations: bool = False + admission_rejected: threading.Event = field(default_factory=threading.Event, + compare=False, repr=False) + execution_unknown: threading.Event = field(default_factory=threading.Event, + compare=False, repr=False) _CURRENT_EXECUTION: ContextVar[ExecutionContext | None] = ContextVar( @@ -169,14 +179,23 @@ class DurableCoordinator: raise CoordinationError("coordination_unavailable", 503) return raw, self._decode(raw) except CoordinationError: + mark_execution_unknown() raise except Exception: + mark_execution_unknown() raise CoordinationError("coordination_unavailable", 503) from None def _change(self, mutate): for _attempt in range(self.cas_attempts): raw, document = self._read() - changed, result = mutate(document) + try: + changed, result = mutate(document) + except CoordinationAdmissionRejected: + mark_admission_rejected() + raise + except CoordinationError: + mark_execution_unknown() + raise if not changed: return result document["revision"] += 1 @@ -186,7 +205,9 @@ class DurableCoordinator: except Exception: # A lost commit response may still have installed this claim. # Do not fall back to an in-memory lock or assume rollback. + mark_execution_unknown() raise CoordinationError("coordination_unavailable", 503) from None + mark_execution_unknown() raise CoordinationError("coordination_unavailable", 503) def claim(self, operation_id: str | None = None, resources=(), maintenance: bool = False, @@ -221,7 +242,7 @@ class DurableCoordinator: def mutate(document): if len(document["claims"]) >= self.max_claims: - raise CoordinationError("coordination_busy", 409) + raise CoordinationClaimRejected("coordination_busy", 409) conflicts = [] for existing in document["claims"].values(): conflict = ( @@ -235,8 +256,9 @@ class DurableCoordinator: conflicts.append(existing) if conflicts: unknown = any(existing["state"] == "uncertain" for existing in conflicts) - raise CoordinationError("coordination_uncertain" if unknown - else "coordination_busy", 503 if unknown else 409) + raise CoordinationClaimRejected( + "coordination_uncertain" if unknown else "coordination_busy", + 503 if unknown else 409) document["claims"][token] = record return True, claim @@ -320,8 +342,9 @@ class DurableCoordinator: and bool(set(combined).intersection(existing["resources"])) ): unknown = existing["state"] == "uncertain" - raise CoordinationError("coordination_uncertain" if unknown - else "coordination_busy", 503 if unknown else 409) + raise CoordinationClaimRejected( + "coordination_uncertain" if unknown else "coordination_busy", + 503 if unknown else 409) changed = list(combined) != current["resources"] current.update(resources=list(combined), updated_at=int(time.time())) return changed, Claim(current["token"], current["execution_id"], @@ -391,7 +414,8 @@ def update_current_claim(claim: Claim) -> None: raise ValueError("Only the current execution claim may be updated") _CURRENT_EXECUTION.set(ExecutionContext( context.coordinator, claim, context.remote_write_started, - context.connection_failure_safe)) + context.connection_failure_safe, context.tracked_mutations, + context.admission_rejected, context.execution_unknown)) def mark_remote_write_started() -> None: @@ -401,6 +425,41 @@ def mark_remote_write_started() -> None: context.remote_write_started.set() +def mark_execution_unknown() -> None: + """Never allow later no-write evidence to erase an ambiguous execution.""" + context = current_execution() + if context is not None: + context.execution_unknown.set() + + +def mark_admission_rejected() -> None: + """Called only by a confirmed local admission boundary before dispatch.""" + context = current_execution() + if context is not None: + context.admission_rejected.set() + + +def execution_failure_before_write(error_code: str | None) -> bool: + """Live no-dispatch evidence for explicitly tracked VM execution only. + + This never reconciles a stored claim. A pending background owner must also + be checked by callers deciding whether the whole request has completed. + An error code cannot itself prove that someone else's fence rejected us. + """ + context = current_execution() + if (context is None or not context.tracked_mutations + or context.remote_write_started.is_set() + or context.execution_unknown.is_set() + or error_code == "coordination_unavailable"): + return False + if error_code in {"coordination_uncertain", "libvirt_execution_blocked"}: + return context.admission_rejected.is_set() + if (error_code in _UNKNOWN_CODES + or isinstance(error_code, str) and error_code.endswith("_uncertain")): + return False + return True + + def connection_failure_before_write(error_code: str) -> bool: """Allow only explicitly libvirt-only scopes to use connection evidence. @@ -411,12 +470,14 @@ def connection_failure_before_write(error_code: str) -> bool: context = current_execution() return bool(error_code == "libvirt_connection_failed" and context is not None and context.connection_failure_safe + and not context.execution_unknown.is_set() and not context.remote_write_started.is_set()) @contextmanager def execution_context(coordinator: DurableCoordinator, claim: Claim, *, - connection_failure_safe: bool | None = None): + connection_failure_safe: bool | None = None, + tracked_mutations: bool | None = None): parent = current_execution() same_execution = (parent is not None and parent.coordinator is coordinator and parent.claim.execution_id == claim.execution_id) @@ -424,7 +485,13 @@ def execution_context(coordinator: DurableCoordinator, claim: Claim, *, safe = (parent.connection_failure_safe if same_execution else False) if connection_failure_safe is not None: safe = connection_failure_safe - token = _CURRENT_EXECUTION.set(ExecutionContext(coordinator, claim, evidence, safe)) + tracked = parent.tracked_mutations if same_execution else False + if tracked_mutations is not None: + tracked = tracked_mutations + rejected = parent.admission_rejected if same_execution else threading.Event() + unknown = parent.execution_unknown if same_execution else threading.Event() + token = _CURRENT_EXECUTION.set(ExecutionContext( + coordinator, claim, evidence, safe, tracked, rejected, unknown)) try: yield finally: @@ -433,11 +500,34 @@ def execution_context(coordinator: DurableCoordinator, claim: Claim, *, def _uncertain_exception(exc: BaseException) -> bool: if isinstance(exc, CoordinationAdmissionRejected): + mark_admission_rejected() context = current_execution() - return bool(context is not None and context.remote_write_started.is_set()) + if (isinstance(exc, CoordinationClaimRejected) and exc.status >= 500 + and not execution_failure_before_write(exc.error_code)): + return True + return bool(context is not None and (context.remote_write_started.is_set() + or context.execution_unknown.is_set())) + context = current_execution() + if context is not None and context.execution_unknown.is_set(): + return True code = getattr(exc, "error_code", None) - if code in _UNKNOWN_CODES: + # Cancellation may race another thread which has not published its write + # marker yet. A disconnected caller is not evidence of no dispatch. + if not isinstance(exc, Exception): + mark_execution_unknown() return True + if isinstance(exc, CoordinationError) and exc.status >= 500: + mark_execution_unknown() + return True + if (code == "coordination_unavailable" + or code in _UNKNOWN_CODES + or isinstance(code, str) and code.endswith("_uncertain")): + if execution_failure_before_write(code): + return False + mark_execution_unknown() + return True + if execution_failure_before_write(code): + return False if code in _SAFE_EXECUTION_CODES: context = current_execution() return bool(context is not None and context.remote_write_started.is_set()) @@ -449,7 +539,8 @@ def _uncertain_exception(exc: BaseException) -> bool: @contextmanager def resource_guard(resources, timeout: float = 5.0, *, - connection_failure_safe: bool | None = None): + connection_failure_safe: bool | None = None, + tracked_mutations: bool | None = None): import math if (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) @@ -472,10 +563,15 @@ def resource_guard(resources, timeout: float = 5.0, *, except CoordinationError as exc: remaining = deadline - time.monotonic() if exc.error_code != "coordination_busy" or remaining <= 0: + if isinstance(exc, CoordinationAdmissionRejected): + mark_admission_rejected() + else: + mark_execution_unknown() raise time.sleep(min(0.02, remaining)) with execution_context(coordinator, claim, - connection_failure_safe=connection_failure_safe): + connection_failure_safe=connection_failure_safe, + tracked_mutations=tracked_mutations): try: yield claim except BaseException as exc: @@ -487,8 +583,10 @@ def resource_guard(resources, timeout: float = 5.0, *, else: settled = coordinator.release(claim) if not settled: + mark_execution_unknown() raise CoordinationError("coordination_uncertain", 503) from exc raise else: if not coordinator.release(claim): + mark_execution_unknown() raise CoordinationError("coordination_uncertain", 503) diff --git a/engine/virsh_panel/durable_jobs.py b/engine/virsh_panel/durable_jobs.py index 6c693ea0..f3182df8 100644 --- a/engine/virsh_panel/durable_jobs.py +++ b/engine/virsh_panel/durable_jobs.py @@ -11,7 +11,8 @@ import threading from typing import Any, Callable, Mapping from .coordination import (CoordinationError, current_execution, execution_context, - connection_failure_before_write) + connection_failure_before_write, + execution_failure_before_write) LOG = logging.getLogger(__name__) @@ -20,12 +21,26 @@ _TERMINAL = frozenset({"completed", "succeeded", "failed"}) def _terminal_outcome(record, coordinator, execution_id: str) -> bool: code = record.get("error_code", "") if isinstance(record, Mapping) else "" + context = current_execution() + if context is not None and context.execution_unknown.is_set(): + return False + # This receipt is produced only by the still-running, instrumented worker. + # A stored flag alone cannot settle old jobs or erase a previous write. + prewrite_failure = ( + isinstance(record, Mapping) + and record.get("status") == "failed" + and record.get("mutation_started") is False + and record.get("execution_outcome") == "not_dispatched" + and execution_failure_before_write(code) + ) return ( isinstance(record, Mapping) and record.get("status") in _TERMINAL and not record.get("cleanup_pending") - and not str(code).startswith(("libvirt_execution_", "coordination_")) - and (code != "libvirt_connection_failed" or connection_failure_before_write(code)) + and (prewrite_failure + or not str(code).startswith(("libvirt_execution_", "coordination_"))) + and (code != "libvirt_connection_failed" or prewrite_failure + or connection_failure_before_write(code)) and not any(claim["execution_id"] == execution_id and claim["state"] == "uncertain" for claim in coordinator.snapshot().values()) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index aa2505e8..d25aabe0 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -53,7 +53,8 @@ from .libvirt_rpc import EXECUTION_ERROR_CODES as LIBVIRT_EXECUTION_ERROR_CODES from .coordination import ( DurableCoordinator, CoordinationError, current_execution, execution_context, configure_default_coordinator, update_current_claim, _uncertain_exception, - connection_failure_before_write, + connection_failure_before_write, execution_failure_before_write, + mark_execution_unknown, ) from .shutdown import (shutdown_scope, remaining_shutdown_seconds, run_bounded_close, install_asyncio_executor, @@ -121,13 +122,22 @@ EXECUTION_UNCERTAIN_ERROR_CODES = frozenset({ }) +def _not_dispatched(record: dict) -> bool: + """Only new, terminal receipts carry affirmative no-dispatch evidence.""" + return (record.get("status") == "failed" + and record.get("mutation_started") is False + and record.get("execution_outcome") == "not_dispatched") + + def _execution_uncertain(record: dict) -> bool: """A stopped local worker is not proof that its remote write stopped.""" - return record.get("error_code") in EXECUTION_UNCERTAIN_ERROR_CODES + return (not _not_dispatched(record) + and record.get("error_code") in EXECUTION_UNCERTAIN_ERROR_CODES) def _retain_execution_uncertainty(record: dict) -> None: - record.update(status="uncertain", safe_to_refund=False) + record.update(status="uncertain", safe_to_refund=False, + execution_outcome="unknown") record.pop("finished_at", None) @@ -434,6 +444,8 @@ class _DurableAsyncLock(asyncio.Lock): not_started = record.get("mutation_started") is False except Exception: pass + if current_execution().execution_unknown.is_set(): + not_started = False if exc is not None and _uncertain_exception(exc) and not not_started: await asyncio.to_thread(self.coordinator.retain, self.claim) else: @@ -871,7 +883,8 @@ class EngineApplication: async def capture(message): messages.append(message) with execution_context(self.coordinator, claim, - connection_failure_safe=libvirt_only): + connection_failure_safe=libvirt_only, + tracked_mutations=bool(operation_request)): try: if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409) @@ -925,11 +938,15 @@ class EngineApplication: or record.get("network_mutation_started") is False or record.get("vm_delete_committed") is True and bool(record.get("cleanup_pending"))) - if (status >= 500 or record.get("status") == "uncertain") and not safe_failure: + if current_execution().execution_unknown.is_set(): + safe_failure = False + if (current_execution().execution_unknown.is_set() + or (status >= 500 or record.get("status") == "uncertain") and not safe_failure): await asyncio.to_thread(self.coordinator.retain, claim) else: await asyncio.to_thread(self.coordinator.release, claim) except BaseException as exc: + from .libvirt_service import VMError not_started = False try: current = (await asyncio.to_thread( @@ -944,6 +961,11 @@ class EngineApplication: raw is not None and json.loads(raw).get("mutation_started") is False) except Exception: pass + if (not not_started and not isinstance( + exc, (VMError, CoordinationError, EngineAuthError))): + mark_execution_unknown() + if current_execution().execution_unknown.is_set(): + not_started = False if _uncertain_exception(exc) and not not_started and dispatch_started: await asyncio.to_thread(self.coordinator.retain, claim) else: @@ -1007,6 +1029,22 @@ class EngineApplication: self.update_coordination_lock = asyncio.Lock() return self.update_coordination_lock + async def _failed_before_dispatch(self, operation, error_code) -> bool: + if error_code in DELETE_PUBLIC_ERROR_CODES | REINSTALL_PUBLIC_ERROR_CODES: + return False + if not execution_failure_before_write(error_code): + return False + # An accepted background job can start after this HTTP handler fails. + # Only that worker can publish its own terminal no-dispatch receipt. + context = current_execution() + claims = await asyncio.to_thread(self.coordinator.snapshot) + if any(row["execution_id"] == context.claim.execution_id + and token != context.claim.token for token, row in claims.items()): + return False + # Public IP reservations have their own durable mutation boundary. + return (operation.get("network_mutation_started") is not True + and execution_failure_before_write(error_code)) + async def _dispatch_regular(self, scope, receive, send, body, delegated, *, existing_only=False): path = scope["path"] if path.startswith("/internal/v1/fabric"): @@ -1211,6 +1249,13 @@ class EngineApplication: child_scope["virt_cloud_mark_network_mutation_started"] = \ mark_network_mutation_started + def mark_network_mutation_not_started(): + updated = {**operation, "network_mutation_started": False} + self.store.set("operation", operation["operation_id"], json.dumps(updated)) + operation.update(network_mutation_started=False) + + child_scope["virt_cloud_mark_network_mutation_not_started"] = \ + mark_network_mutation_not_started if operation is None: # Register before saving so all persisted hosts are restored for # unattended sampling after restart, even if the HTTP caller dies. @@ -1248,6 +1293,7 @@ class EngineApplication: # PHP additionally requires network_mutation_started for that # resource before gating failed results. operation["mutation_started"] = True + operation["execution_outcome"] = "unknown" await asyncio.to_thread( self.store.set, "operation", operation["operation_id"], json.dumps(operation)) @@ -1265,12 +1311,15 @@ class EngineApplication: if context else {}) failed_before_start = admission.get("metadata", {}).get("failed_before_start") is True connection_failed = connection_failure_before_write(payload.get("error_code")) + not_dispatched = (status >= 400 and await self._failed_before_dispatch( + operation, payload.get("error_code"))) operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else ( - "failed" if not create and ( + "failed" if not_dispatched or not create and ( status < 500 or network_preflight_failed or failed_before_start or connection_failed) else "uncertain") if operation["status"] == "succeeded": operation["result"] = payload + operation["execution_outcome"] = "acknowledged" elif operation["status"] == "pending": operation["result"] = payload operation["job_id"] = payload["job_id"] @@ -1289,7 +1338,7 @@ class EngineApplication: or (network_preflight_failed and error_code in FIREWALL_PUBLIC_ERROR_CODES)): operation["error_code"] = error_code - if error_code in EXECUTION_ERROR_CODES: + if error_code in EXECUTION_ERROR_CODES and not not_dispatched: operation["status"] = "uncertain" if self._is_vm_delete_operation(operation): cleanup_code = payload.get("error_code") @@ -1300,10 +1349,14 @@ class EngineApplication: # finish exact cleanup without replaying VM DELETE. operation["cleanup_pending"] = "vm_delete" operation["vm_delete_committed"] = True + if not_dispatched: + operation.update(status="failed", mutation_started=False, + execution_outcome="not_dispatched") if create: await self._check_absent(operation) operation["finished_at"] = int(time.time()) - if _execution_uncertain(operation): + if (_execution_uncertain(operation) + or current_execution() and current_execution().execution_unknown.is_set()): _retain_execution_uncertainty(operation) await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation)) if operation["status"] in {"succeeded", "pending"}: @@ -1320,10 +1373,24 @@ class EngineApplication: ) preflight_status = getattr(exc, "status", None) error_code = getattr(exc, "error_code", "") - if error_code in EXECUTION_ERROR_CODES: + from .libvirt_service import VMError + # Application/database failures are not VM transport receipts. + # Never infer no-dispatch from an unhandled persistence exception. + not_dispatched = ( + isinstance(exc, (VMError, CoordinationError)) + and await self._failed_before_dispatch(operation, error_code)) + if not_dispatched: + operation.update(status="failed", mutation_started=False, + execution_outcome="not_dispatched", error=str(exc), + finished_at=int(time.time())) + if error_code in EXECUTION_ERROR_CODES: + operation["error_code"] = error_code + if create: + await self._check_absent(operation) + elif error_code in EXECUTION_ERROR_CODES: operation.update( status="uncertain", error=str(exc), error_code=error_code, - safe_to_refund=False, + safe_to_refund=False, execution_outcome="unknown", ) operation.pop("finished_at", None) elif (isinstance(preflight_status, int) @@ -1341,6 +1408,7 @@ class EngineApplication: ) else: operation["status"] = "uncertain" + operation["execution_outcome"] = "unknown" operation["error"] = ( "防火墙结果不确定,请查询原操作核对,不要盲目重放" if self._is_firewall_resource(operation.get("resource", "")) @@ -2452,7 +2520,8 @@ class EngineApplication: if error: # No service, task, VM or volume mutation has been invoked. record.update(status="failed", prevalidation_failed=True, - error=error, safe_to_refund=create, finished_at=int(time.time())) + error=error, safe_to_refund=create, finished_at=int(time.time()), + execution_outcome="not_dispatched") if self._is_firewall_resource(resource): record["error_code"] = "firewall_rule_invalid" if existing_only: @@ -2689,6 +2758,8 @@ class EngineApplication: ) if job.get("owner") != f"php:{record['user_id']}" or job.get("vm_id") != record["vm_uuid"]: return record + if job.get("operation_id") != record["operation_id"]: + return record if (kind == "snapshot_job" and job.get("status") == "uncertain" and not _execution_uncertain(job)): reconciler = getattr(app.state, "reconcile_snapshot_job", None) @@ -2739,6 +2810,12 @@ class EngineApplication: kind == "reinstall_job" and _reinstall_committed(job) ) + if _not_dispatched(job) and not committed_reinstall: + record.update(mutation_started=False, execution_outcome="not_dispatched") + elif record["status"] == "succeeded": + record["execution_outcome"] = "acknowledged" + else: + record["execution_outcome"] = "unknown" if (committed_reinstall and record["status"] in {"queued", "running", "succeeded", "failed"}): # A progress marker is written after the replacement domain has @@ -3791,6 +3868,11 @@ class EngineApplication: profile, operation["vm_uuid"]) operation.update(status="failed", safe_to_refund=True) except Exception as exc: + if _not_dispatched(operation): + # Execution is known not to have dispatched; absence and + # local allocation cleanup still need independent proof + # before PHP may refund this purchase. + return error_code = getattr(exc, "error_code", "") if error_code in EXECUTION_ERROR_CODES: operation.update(error_code=error_code, error=str(exc)) @@ -3813,6 +3895,8 @@ class EngineApplication: "operation_id", "status", "result", "error", "safe_to_refund", "service_was_running_hint", "created_at", "finished_at") if key in record} + if record.get("execution_outcome") in {"not_dispatched", "acknowledged", "unknown"}: + result["execution_outcome"] = record["execution_outcome"] if record.get("error_code") in EXECUTION_ERROR_CODES: result["error_code"] = record["error_code"] # Expose only bounded lifecycle markers. Do not leak persisted diff --git a/engine/virsh_panel/libvirt_rpc.py b/engine/virsh_panel/libvirt_rpc.py index 5cf26e43..fc65d71f 100644 --- a/engine/virsh_panel/libvirt_rpc.py +++ b/engine/virsh_panel/libvirt_rpc.py @@ -321,16 +321,18 @@ class LibvirtRuntime: os.close(fd) def _before_write(self, session, method): - from .coordination import mark_remote_write_started + from .coordination import mark_admission_rejected, mark_remote_write_started with self._condition: if self._closing: raise ExecutionFailure("libvirt_execution_stopped") if session.scope in self._snapshot_recoveries: + mark_admission_rejected() raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) directory = self._directory(session.scope) active = {s.token for s in self._sessions if s.failure is None} if any(path.stem not in active for path in directory.glob("*.json")): + mark_admission_rejected() raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) if session.intent is not None: mark_remote_write_started() @@ -530,6 +532,8 @@ class RPCSession: return self.process.pid if self.process is not None else None def _fail(self, code, *, outcome_unknown=False): + from .coordination import execution_failure_before_write, mark_execution_unknown + # Intent persistence and shutdown share this condition. A later local # stop/error must never downgrade evidence of an unconfirmed write. with self.runtime._condition: @@ -548,6 +552,9 @@ class RPCSession: if (previous is None or previous.error_code != code or previous.outcome_unknown != outcome_unknown): self.failure = ExecutionFailure(code, outcome_unknown=outcome_unknown) + if outcome_unknown and not (code == "libvirt_execution_blocked" + and execution_failure_before_write(code)): + mark_execution_unknown() return self.failure def start(self): diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index d549ac13..b50b0af7 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -874,7 +874,10 @@ class VMService: with resource_guard(resources, timeout=max( 0.0, deadline - time.monotonic()), connection_failure_safe=(current_execution() is None - or current_execution().connection_failure_safe)): + or current_execution().connection_failure_safe), + tracked_mutations=(self.execution_scope is not None + and (current_execution() is None + or current_execution().tracked_mutations))): with local_guard(): yield except CoordinationError as exc: diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 1e7f0c8d..29513a8d 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -66,6 +66,38 @@ final class CloudWorkerService return in_array($status, ['pending', 'running'], true) ? 'running' : 'uncertain'; } + /** Only a bound engine receipt can undo this request's local delete latch. */ + private static function confirmedUndispatchedFailure( + CloudJob $job, array $operation + ): bool { + if (($operation['status'] ?? null) !== 'failed' + || ($operation['execution_outcome'] ?? null) !== 'not_dispatched' + || ($operation['mutation_started'] ?? null) !== false + || ($operation['executor_active'] ?? false) === true + || !is_string($operation['operation_id'] ?? null) + || !hash_equals((string) $job->operation_id, $operation['operation_id'])) { + return false; + } + // Contradictory post-commit evidence remains subject to review. This + // receipt says only that this request did not mutate the guest; it is + // not a proof of guest absence and is never used to authorize refunds. + $result = is_array($operation['result'] ?? null) ? $operation['result'] : []; + $vm = is_array($result['vm'] ?? null) ? $result['vm'] : []; + foreach ([$operation, $result, $vm] as $evidence) { + if (array_key_exists('cleanup_pending', $evidence)) { + return false; + } + foreach (['vm_delete_committed', 'network_mutation_started', 'deleted', + 'created', 'restored', 'reinstall_committed', 'network_changed', + 'settings_changed', 'command_accepted'] as $key) { + if (($evidence[$key] ?? false) === true) { + return false; + } + } + } + return true; + } + /** * A reinstall can commit the replacement guest before old-volume cleanup * is confirmed. Preserve the new credential/spec while keeping the job @@ -913,9 +945,11 @@ final class CloudWorkerService $instance = CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); if ($instance !== null && (string) $instance->status === 'deleting') { - // Keep a failed destructive request visible and block - // new VM/VPC mutations until an operator reconciles it. - $instance->status = 'delete_failed'; + // A proven preflight/admission rejection did not touch + // this VM. Release only this user-delete admission latch; + // unknown/legacy deletes still require operator review. + $instance->status = self::confirmedUndispatchedFailure($job, $operation) + ? 'active' : 'delete_failed'; $instance->updated_at = time(); $instance->save(); } diff --git a/web/tests/CloudPowerReceiptWorkflowTest.php b/web/tests/CloudPowerReceiptWorkflowTest.php index c6f4e1b4..442c4d4d 100644 --- a/web/tests/CloudPowerReceiptWorkflowTest.php +++ b/web/tests/CloudPowerReceiptWorkflowTest.php @@ -31,7 +31,8 @@ namespace App\Services\Cloud { array $actor, ?array $body, string $operation): array { $this->calls[] = [$host, $method, $resource, $body]; - if (!str_ends_with($resource, '/actions')) { + if (!str_ends_with($resource, '/actions') + && !str_ends_with($resource, '/settings') && $method !== 'DELETE') { throw new CloudError('Private network/QGA unavailable', 503); } if ($this->afterCommand !== null) { ($this->afterCommand)(); } @@ -95,6 +96,11 @@ namespace { $t->integer($key); } }); + $schema->create('cloud_instance_snapshots', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('instance_id'); $t->string('state'); + $t->string('last_operation_id')->nullable(); $t->integer('deleted_at')->nullable(); + $t->integer('updated_at')->default(0); + }); $schema->create('cloud_vpcs', static function (Blueprint $t): void { $t->increments('id'); $t->string('vpc_key'); $t->integer('desired_revision'); $t->string('desired_digest'); $t->string('state'); $t->string('sync_state'); @@ -195,6 +201,90 @@ namespace { // Isolate the following rejection cases; no production cancellation code is used. $next->status = 'cancelled'; $next->save(); + // An engine-side preflight/admission failure is terminal even though PHP + // sent HTTP and the VM has an unavailable private network. No guest write + // happened, so it must not dispatch VPC reconciliation or block force-off. + foreach (['action', 'instance_settings'] as $kind) { + $job = $kind === 'action' ? $enqueue('reboot') + : CloudJobService::enqueue(CloudInstance::find(10), $actor, + 'instance_settings', 'POST', '/settings', [ + 'hostname' => 'preflight-test', 'confirm_restart' => true, + ]); + $engine->response = ['status' => 'failed', + 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, + 'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain']; + $count = count($engine->calls); + $worker->tick(); + $failed = CloudJob::find($job->id); + $assert($failed->status === 'failed' && $failed->lease_token === null + && (int) $failed->lease_until === 0 + && count($engine->calls) === $count + 1, + 'Proven preflight failure acquired a private-state dependency for ' . $kind . '.'); + $followup = $enqueue('poweroff'); + $engine->response = ['status' => 'succeeded', 'result' => [ + 'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true, + ]]; + $worker->tick(); + $assert(CloudJob::find($followup->id)->status === 'succeeded', + 'Preflight failure blocked the next force-off request for ' . $kind . '.'); + } + + // User deletion has a separate instance admission latch. Undo it only + // for this operation's explicit no-write receipt, never a legacy or + // contradictory outcome; preserve the snapshot catalog in all failures. + foreach ([[], ['execution_outcome' => null], ['mutation_started' => true], + ['operation_id' => str_repeat('f', 32)], ['executor_active' => true], + ['vm_delete_committed' => true], ['result' => ['cleanup_pending' => true]]] + as $index => $change) { + $id = 11 + $index; + $targetUuid = sprintf('22222222-2222-4222-8222-%012d', $id); + Capsule::table('cloud_instances')->insert(['id' => $id, 'host_id' => 7, + 'user_id' => 1, 'uuid' => $targetUuid, 'name' => 'Delete preflight']); + Capsule::table('cloud_instance_snapshots')->insert([ + 'instance_id' => $id, 'state' => 'ready']); + $delete = CloudJobService::enqueue(CloudInstance::find($id), $actor, + 'delete', 'DELETE', '?delete_disks=true'); + $assert(CloudInstance::find($id)->status === 'deleting', + 'Delete request did not establish its admission latch.'); + $engine->response = array_replace(['status' => 'failed', + 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, + 'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'], $change); + $worker->tick(); + $assert(CloudJob::find($delete->id)->status === 'failed' + && CloudInstance::find($id)->status === ($index === 0 ? 'active' : 'delete_failed') + && Capsule::table('cloud_instance_snapshots')->where('instance_id', $id) + ->value('state') === 'ready', + 'User-delete latch ignored missing/conflicting execution proof.'); + if ($index === 0) { + $next = CloudJobService::enqueue(CloudInstance::find($id), $actor, + 'action', 'POST', '/actions', ['action' => 'poweroff']); + $next->status = 'cancelled'; $next->save(); + } else { + try { + CloudJobService::enqueue(CloudInstance::find($id), $actor, + 'action', 'POST', '/actions', ['action' => 'poweroff']); + throw new RuntimeException('Unproven failed delete admitted another write.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 409, 'Unproven delete had unexpected admission.'); + } + } + } + + // Expiry cleanup owns a separate service-policy latch. A no-write failure + // must not restore an expired instance to an active service. + Capsule::table('cloud_instances')->insert(['id' => 30, 'host_id' => 7, + 'user_id' => 1, 'uuid' => '22222222-2222-4222-8222-000000000030', + 'name' => 'Expiry preflight', 'status' => 'deleting', + 'service_status' => 'purge_deleting']); + $expiry = CloudJobService::enqueueExpiryDelete(CloudInstance::find(30)); + $engine->response = ['status' => 'failed', 'execution_outcome' => 'not_dispatched', + 'mutation_started' => false, 'safe_to_refund' => false]; + $worker->tick(); + $assert(CloudJob::find($expiry->id)->status === 'failed' + && CloudInstance::find(30)->status === 'delete_failed' + && CloudInstance::find(30)->service_status === 'purge_failed', + 'Preflight failure reopened an expired service instead of preserving its policy latch.'); + $gate = new ReflectionMethod(CloudWorkerService::class, 'applyPrivateLifecycleGate'); $confirmed = new ReflectionMethod(CloudWorkerService::class, 'confirmedPowerReceipt'); $job = $enqueue('shutdown'); @@ -229,6 +319,7 @@ namespace { // A lost power response is still uncertain, and the next write is rejected. $engine->response = ['status' => 'uncertain', 'mutation_started' => true, + 'execution_outcome' => 'unknown', 'error_code' => 'libvirt_execution_uncertain']; $worker->tick(); $assert(CloudJob::find($job->id)->status === 'uncertain', diff --git a/web/tests/CloudPrewriteFailureWorkflowTest.php b/web/tests/CloudPrewriteFailureWorkflowTest.php new file mode 100644 index 00000000..477e9bb8 --- /dev/null +++ b/web/tests/CloudPrewriteFailureWorkflowTest.php @@ -0,0 +1,167 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema(); + $schema->create('user', static function (Blueprint $t): void { + $t->increments('id'); $t->string('money'); $t->integer('enable')->default(1); + $t->boolean('is_admin')->default(false); + }); + $schema->create('cloud_hosts', static function (Blueprint $t): void { + $t->increments('id'); + }); + $schema->create('cloud_offers', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('stock'); $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_orders', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('user_id'); $t->integer('host_id'); + $t->integer('offer_id'); $t->string('amount'); $t->string('status'); + $t->integer('stock_reserved_at'); $t->integer('stock_released_at')->nullable(); + $t->integer('refunded_at')->nullable(); $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_invoices', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('order_id'); $t->string('purpose'); + $t->string('status'); $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_resource_reservations', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('order_id'); $t->integer('host_id'); + $t->integer('public_ip_id'); $t->integer('network_attachment_id'); + $t->integer('public_ip_pool_id'); $t->string('operation_id'); + $t->string('state'); $t->integer('released_at')->nullable(); + $t->string('release_reason')->nullable(); $t->integer('expires_at')->nullable(); + $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_public_ips', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('host_id'); $t->integer('public_ip_pool_id'); + $t->integer('reservation_id')->nullable(); $t->integer('instance_id')->nullable(); + $t->string('operation_id')->nullable(); $t->string('state'); + $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_jobs', static function (Blueprint $t): void { + $t->increments('id'); $t->string('operation_id')->unique(); + $t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable(); + $t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id'); + $t->string('kind'); $t->string('status'); $t->text('payload_cipher'); + $t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable(); + $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); + $t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at'); + $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); + }); + Capsule::table('user')->insert(['id' => 1, 'money' => '100.00']); + Capsule::table('cloud_hosts')->insert(['id' => 7]); + Capsule::table('cloud_offers')->insert(['id' => 1, 'stock' => 10]); + $history = []; + $mock = new MockHandler(); + $handler = HandlerStack::create($mock); + $handler->push(Middleware::history($history)); + $worker = new CloudWorkerService(new CloudEngineClient(new Client(['handler' => $handler]))); + // Begin at the paid, reserved boundary. All subsequent transitions use + // the real worker, finance services, Eloquent and HTTP outcome validation. + $paidCreate = static function (int $id): CloudJob { + $operation = bin2hex(random_bytes(16)); + $now = time(); + Capsule::table('user')->where('id', 1)->decrement('money', 9.99); + Capsule::table('cloud_offers')->where('id', 1)->decrement('stock'); + Capsule::table('cloud_orders')->insert(['id' => $id, 'user_id' => 1, + 'host_id' => 7, 'offer_id' => 1, 'amount' => '9.99', 'status' => 'queued', + 'stock_reserved_at' => $now]); + Capsule::table('cloud_invoices')->insert(['id' => $id, 'order_id' => $id, + 'purpose' => 'provision', 'status' => 'paid']); + Capsule::table('cloud_resource_reservations')->insert(['id' => $id, + 'order_id' => $id, 'host_id' => 7, 'public_ip_id' => $id, + 'network_attachment_id' => 1, 'public_ip_pool_id' => 1, + 'operation_id' => $operation, 'state' => 'held', 'expires_at' => $now + 3600]); + Capsule::table('cloud_public_ips')->insert(['id' => $id, 'host_id' => 7, + 'public_ip_pool_id' => 1, 'reservation_id' => $id, + 'operation_id' => $operation, 'state' => 'reserved']); + Capsule::table('cloud_jobs')->insert(['id' => $id, 'operation_id' => $operation, + 'order_id' => $id, 'user_id' => 1, 'host_id' => 7, + 'kind' => 'create', 'status' => 'queued', 'payload_cipher' => CloudSecrets::encrypt([ + 'method' => 'POST', 'resource' => 'vms', 'body' => ['name' => 'preflight-' . $id], + ]), 'available_at' => $now, 'created_at' => $now, 'updated_at' => $now]); + return CloudJob::find($id); + }; + $job = $paidCreate(1); + $mock->append(new Response(503, [], json_encode([ + 'operation_id' => $job->operation_id, 'status' => 'failed', + 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, + 'safe_to_refund' => true, 'error_code' => 'execution_result_uncertain', + ]))); + $assert($worker->tick(), 'Preflight failure was not consumed.'); + $assert(CloudJob::find(1)->status === 'failed' + && (string) Capsule::table('user')->value('money') === '100.00' + && Capsule::table('cloud_offers')->value('stock') === 10 + && Capsule::table('cloud_orders')->where('id', 1)->value('refunded_at') !== null + && Capsule::table('cloud_invoices')->where('id', 1)->value('status') === 'refunded' + && Capsule::table('cloud_resource_reservations')->where('id', 1)->value('state') === 'released' + && Capsule::table('cloud_public_ips')->where('id', 1)->value('state') === 'available', + 'Proven no-write 503 retained a create lock, wallet charge, stock or IP.'); + $assert(!$worker->tick() && count($history) === 1 + && (string) Capsule::table('user')->value('money') === '100.00', + 'Terminal preflight failure was dispatched or refunded twice.'); + + foreach ([['unknown', 'uncertain', true], ['not_dispatched', 'failed', false]] + as $index => [$outcome, $status, $started]) { + $id = $index + 2; + $job = $paidCreate($id); + $mock->append(new Response(503, [], json_encode([ + 'operation_id' => $job->operation_id, 'status' => $status, + 'execution_outcome' => $outcome, 'mutation_started' => $started, + 'safe_to_refund' => false, 'error_code' => 'libvirt_execution_uncertain', + ]))); + $worker->tick(); + $assert(CloudJob::find($id)->status === 'uncertain' + && Capsule::table('cloud_orders')->where('id', $id)->value('refunded_at') === null + && Capsule::table('cloud_invoices')->where('id', $id)->value('status') === 'paid' + && Capsule::table('cloud_resource_reservations')->where('id', $id)->value('state') === 'uncertain' + && Capsule::table('cloud_public_ips')->where('id', $id)->value('state') === 'uncertain' + && Capsule::table('cloud_offers')->value('stock') === 11 - $id, + 'Unknown write or missing absence proof released a create reservation.'); + } + $assert((string) Capsule::table('user')->value('money') === '80.02', + 'Unproven creation outcomes refunded the wallet.'); + echo "Prewrite creation failure workflow tests passed.\n"; +}