diff --git a/docs/ENGINE_COORDINATION.md b/docs/ENGINE_COORDINATION.md index 93430e5a..8492a955 100644 --- a/docs/ENGINE_COORDINATION.md +++ b/docs/ENGINE_COORDINATION.md @@ -44,6 +44,28 @@ HTTP 请求的实际执行由受跟踪任务持有,调用者取消只停止等待;原执行继续保存结果后再释放资源。共享停机预算耗尽时保留未完成占用。普通请求的占用覆盖最终 operation 落库,不能在 libvirt 返回后、结果尚未保存时提前放行。 +### 普通电源命令与状态观察(2026-10-13) + +`POST vms/{uuid}/actions` 的 `start/shutdown/reboot/poweroff/reset` 在 libvirt +调用成功返回后输出 `{uuid, action, command_accepted: true}`。这是命令回执, +不附带或推断实时 `state`,不等待来宾关机、启动完成或 QGA 就绪。 +对应 operation 持久化成功后正常释放本次 VM 占用;相同 operation_id +重试只读取原回执。真正缺少写入回执的操作仍保留 uncertain,不自动重放。 + +PHP 仅对与原加密请求 UUID、action、POST 资源路径及操作编号相符的成功回执, +跳过私网生命周期执行与 VPC expectation 完成门禁。电源命令不推进 VPC 的 +desired/applied/usable,也不改变到期/流量断网策略;独立 VPC 协调继续核验 +宿主和来宾。其他任务的私网、停机、资源清理校验保持不变。 + +用户页面在任务确认后释放按钮,再独立刷新实时电源状态;来宾忽略正常关机时 +允许提交强制断电。慢查询或查询失败不改写命令结果,新命令使旧观察失效。 +救援预挂载后的开机同样只返回回执,救援状态 GET 通过实际 XML 单独确认进入救援。 +本次不迁移或释放历史 uncertain 任务、协调占用及 RPC 未决标记。 + +依据:[virDomainShutdown](https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainShutdown) +与 [virDomainReboot](https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainReboot)。 +来宾可以忽略请求,命令成功与来宾实际状态应分开处理;删除、重装等仍须自行确认停机。 + 相同操作编号的重试仍校验权限和请求摘要。原占用已存在而 operation 尚未入库时返回 409;已有记录则只返回原执行状态。未知执行返回 uncertain、`safe_to_refund=false`,不自动重放或依据瞬时 VM 缺失退款/释放 IP。`executor_active` 是当前 worker 的兼容提示,false 不证明另一 worker 或远端动作已经停止。 排队快照仅在本 worker 的 `Future.cancel()` 证明确实未开始后才能被到期断网取消;必须完成原有持久取消凭据和目录回滚,再取得 VM 资源。不能根据另一 worker 的 queued 字段猜测取消成功。初始任务保存/提交失败的明确执行前凭据允许释放本次占用;数据库提交状态不明仍保留保护。 diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index 33f6c0d9..07cd8dab 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -368,6 +368,13 @@ active/usable 则只由自己的 current-generation QGA 证据推进。一台关 agent 不可用的 VM 不得抹掉同 VPC 其他已验证成员的可用状态,但 VPC 聚合健康仍可显示 degraded,周期 reconcile 会继续推进尚未就绪的成员。 +普通电源动作与上述私网可用性分开完成:`actions` 接口返回匹配原请求 +UUID/action 的 `command_accepted: true` 成功回执后,PHP 完成电源任务, +不等待 QGA、VPC revision 或私网健康,也不由该任务触发私网写入。 +电源状态走独立实时观察;VPC 继续由引擎周期协调恢复和核验,命令回执不把 +成员标为 usable。写回执未知仍阻止新写入,删除、重装、网络策略等任务的 +私网与资源证明要求不变。详见 [普通电源命令与状态观察](ENGINE_COORDINATION.md#普通电源命令与状态观察2026-10-13)。 + 历史 seed 中逐成员 `/32` route 只用于旧 generation;运行中成员保持 `vpc_private_guest_restart_required`,完全停机后的 reconcile 才迁移为单条 VPC CIDR。 4096 成员上限保护记录和回读大小,但普通周期 reconcile 在进入并发 QGA 阶段前仍会按宿主 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 647a0da1..25b10a25 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,45 @@ # 本版验证记录 +## 2026-10-13:普通电源命令回执与状态观察分离 + +- `start/shutdown/reboot/poweroff/reset` 的 libvirt 调用成功后返回 + `{uuid, action, command_accepted: true}`,不再等待最终电源状态或读取展示数据。 + 来宾忽略正常关机时任务仍完成,后续强制断电可接纳;真实回执丢失仍为未知结果, + 不从瞬时状态推断成功,也不自动重放。 +- PHP 仅对绑定原 POST actions 路径、UUID、action 和操作编号的确认回执,跳过 + 私网协调及 VPC expectation 完成门禁;不伪造 VPC 可用性、不改变到期/流量策略。 + VPC 仍由独立周期协调核验。删除、重装、网络及服务策略的证明要求不变。 +- 页面确认任务后释放按钮,实际状态独立刷新;移除专用 60 秒关机等待,新命令使 + 旧状态请求失效。救援预挂载后的开机回执也不承担救援启动证明,由救援 GET 回读。 + 静态资源版本为 `2026101301`。本次没有旧任务、协调锁或 RPC 标记的迁移/自动解锁。 +- Python 相关回归合计 **571 passed、81 subtests passed、3 failed**: + `test_libvirt_service.py`、`test_resource_locks.py` 为 256 passed / 81 subtests; + `test_app.py`、`test_engine_boundary.py`、`test_engine_coordination.py`、 + `test_app_vm_concurrency.py` 为 265 passed / 3 failed; + `test_libvirt_service_rpc.py`、`test_durable_coordination.py`、 + `test_libvirt_rpc_failures.py` 为 50 passed。 + 三项失败已在修改前 `c9f2fdb9` 独立源码快照以相同依赖逐项复现: + `test_cloud_template_import_job_and_create_dispatch`(策略快照旧断言)、 + `test_reinstall_cleanup_pending_is_not_reported_success`(错误字段旧断言)、 + `test_fabric_is_engine_global_and_super_admin_write_only`(Fabric readiness)。 + 本次未放宽或修改这些断言。强化后的救援 HTTP 用例另行通过,不重复计数。 +- 新增 `CloudPowerReceiptWorkflowTest.php` 使用 PHP 8.5 WASM、SQLite/Eloquent + 与真实 worker/任务准入服务,覆盖五种电源回执、失效 QGA/私网、revision 变化、 + 后续强制断电、未知回执仍阻断、错误身份不豁免和其他生命周期门禁。 + `CloudPowerActionBehaviorTest.mjs` 执行真实前端函数,覆盖按钮立即释放、 + 慢查询、失败查询、迟到响应隔离与未知任务提示;上述两项及 + `CloudInstanceReadPerformanceTest.php` 均通过。 +- 其余 PHP 回归 `CloudOperationErrorCatalogTest.php`、`CloudInstanceMetadataTest.php`、 + `CloudVpcTombstoneConvergenceTest.php`、`CloudVpcControlPlaneContractTest.php` 通过。 + 扩展检查另有三项既有失败,已用 `c9f2fdb9` 源码及相同依赖复现相同失败点: + `CloudFinancialWorkflowTest.php:504`(reconciliation/email 断言)、 + `CloudBillingLifecycleTest.php:838`(未知删除与策略串行断言)、 + `CloudProtocolTest.php:60`(缺少 `CloudOperationErrorCatalog` 导入)。 + 本次未修复这些无关失败,未宣称完整 PHP 测试集通过。 +- 验证使用模拟 libvirt、内存/SQLite 数据库,没有连接生产宿主、部署或重启服务。 + 不代表真实 libvirt、QGA、MariaDB 并发或浏览器端到端验收。 + API 依据与契约详见 [执行协调说明](ENGINE_COORDINATION.md)。 + ## 2026-10-12:VPC 添加实例候选归属修复 - 根因是用户页面仅排除当前 VPC 的成员,实例列表未携带其他 VPC 的归属。 diff --git a/engine/tests/test_app.py b/engine/tests/test_app.py index 1243fd40..db0e5371 100644 --- a/engine/tests/test_app.py +++ b/engine/tests/test_app.py @@ -388,7 +388,7 @@ class FakeVMService: self.vm["state"] = "running" self.vm["rescue_mode"] = True self.vm["rescue_staged"] = False - return self.vm + return {"uuid": vm_id, "action": action, "command_accepted": True} def service_suspend(self, vm_id): self.calls.append(("service_suspend", vm_id)) @@ -1467,6 +1467,9 @@ def test_vm_crud_actions_and_error_mapping(web): assert created.status_code == 201 and created.json()["uuid"] == VM_ID action = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "reboot"}, headers=headers) assert action.status_code == 200 + assert action.json() == { + "uuid": VM_ID, "action": "reboot", "command_accepted": True, + } deleted = client.delete(f"/api/v1/vms/{VM_ID}?delete_disks=true", headers=headers) assert deleted.status_code == 200 and deleted.json()["deleted"] is True assert service.calls == [ @@ -1643,8 +1646,29 @@ def test_rescue_iso_upload_enter_exit_and_safe_lifecycle(web): assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=json_headers).status_code == 200 assert client.get(rescue_url).json()["staged"] is True - assert client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "start"}, - headers=json_headers).status_code == 200 + original_action, original_get_vm = service.action, service.get_vm + command_accepted = False + + def start_receipt(vm_id, action): + nonlocal command_accepted + receipt = original_action(vm_id, action) + command_accepted = True + return receipt + + def preflight_only(vm_id): + if command_accepted: + raise VMError("rescue boot observation temporarily unavailable", 502) + return original_get_vm(vm_id) + + with ( + patch.object(service, "action", side_effect=start_receipt), + patch.object(service, "get_vm", side_effect=preflight_only), + ): + started = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "start"}, + headers=json_headers) + assert started.status_code == 200, started.text + assert started.json() == {"uuid": VM_ID, "action": "start", "command_accepted": True} + # The next observation, rather than the power receipt, proves rescue boot. assert client.get(rescue_url).json()["active"] is True assert client.get(rescue_url).json()["staged"] is False restored = client.post(f"{rescue_url}/exit", diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index f27cd66f..1a388d80 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -2509,6 +2509,41 @@ def test_power_action_has_persistent_idempotency(boundary): assert operation["mutation_started"] is True +def test_acknowledged_shutdown_releases_target_before_guest_stops(boundary): + client, engine, store, _, services = boundary + call(client, "GET", "/internal/v1/hosts/hk/vms") + components = engine.registry.components["hk"] + components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41) + service = services["hk"] + path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" + receipt = {"uuid": VM_ID, "action": "shutdown", "command_accepted": True} + actor = {**USER, "operation_id": "accepted-shutdown"} + + response = call(client, "POST", path, actor, {"action": "shutdown"}) + assert response.status_code == 200, response.text + assert response.json()["status"] == "succeeded" + assert response.json()["result"] == receipt + assert service.vm["state"] == "running" # This guest ignores shutdown. + assert engine.coordinator.operation_claim(actor["operation_id"]) is None + + # Poll/retry returns the durable receipt even when live observation fails. + with patch.object(service, "get_vm", side_effect=VMError("observation unavailable", 502)), \ + patch.object(service, "get_vm_state", side_effect=VMError("observation unavailable", 502)): + operation = call(client, "GET", "/internal/v1/operations/accepted-shutdown", USER) + assert operation.status_code == 200 + assert operation.json()["result"] == receipt + retried = call(client, "POST", path, actor, {"action": "shutdown"}) + assert retried.status_code == 200 + assert len([row for row in service.calls if row[0] == "action"]) == 1 + + forced = call(client, "POST", path, {**USER, "operation_id": "next-poweroff"}, + {"action": "poweroff"}) + assert forced.status_code == 200, forced.text + assert forced.json()["status"] == "succeeded" + assert ("action", VM_ID, "poweroff") in service.calls + assert engine.coordinator.operation_claim("next-poweroff") is None + + def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 8b48e8c9..e76a8821 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -5001,7 +5001,8 @@ class VMServiceTests(unittest.TestCase): self.assertTrue(domain.isPersistent()) self.assertEqual(len(self.backend.domains), 1) self.assertEqual(self.service.get_vm(vm["uuid"])["state"], "shutoff") - self.assertEqual(self.service.action(vm["uuid"], "start")["state"], "running") + self.assertTrue(self.service.action(vm["uuid"], "start")["command_accepted"]) + self.assertEqual(self.service.get_vm(vm["uuid"])["state"], "running") booted_root = ET.fromstring(domain.XMLDesc(0)) self.assertEqual([boot.get("dev") for boot in booted_root.findall("./os/boot")], ["hd"]) @@ -5471,109 +5472,131 @@ class VMServiceTests(unittest.TestCase): with self.assertRaises(self.error_type) as running: self.service.delete_vm(vm_uuid, True) self.assertEqual(running.exception.status, 409) - self.assertEqual(self.service.action(vm_uuid, "reset")["state"], "running") + self.assertEqual(self.service.action(vm_uuid, "reset"), { + "uuid": vm_uuid, "action": "reset", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "running") self.assertEqual(self.backend.domains[0].last_reset_flags, 0) - self.assertEqual(self.service.action(vm_uuid, "reboot")["state"], "shutoff") - self.assertEqual(self.service.action(vm_uuid, "start")["state"], "running") - self.assertEqual(self.service.action(vm_uuid, "shutdown")["state"], "shutoff") + self.assertEqual(self.service.action(vm_uuid, "reboot"), { + "uuid": vm_uuid, "action": "reboot", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "shutoff") + self.assertEqual(self.service.action(vm_uuid, "start"), { + "uuid": vm_uuid, "action": "start", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "running") + self.assertEqual(self.service.action(vm_uuid, "shutdown"), { + "uuid": vm_uuid, "action": "shutdown", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "shutoff") with self.assertRaises(self.error_type): self.service.console_port(vm_uuid) - self.assertEqual(self.service.action(vm_uuid, "start")["state"], "running") - self.assertEqual(self.service.action(vm_uuid, "poweroff")["state"], "shutoff") + self.assertEqual(self.service.action(vm_uuid, "start"), { + "uuid": vm_uuid, "action": "start", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "running") + self.assertEqual(self.service.action(vm_uuid, "poweroff"), { + "uuid": vm_uuid, "action": "poweroff", "command_accepted": True, + }) + self.assertEqual(self.service.get_vm(vm_uuid)["state"], "shutoff") result = self.service.delete_vm(vm_uuid, True) self.assertEqual(result["deleted_disks"], ["alma-test.qcow2"]) self.assertFalse(self.backend.pools["images"].volumes) self.assertFalse(self.backend.domains) - def test_power_action_does_not_report_graceful_shutdown_before_shutoff(self): + def test_shutdown_receipt_does_not_require_shutoff_and_allows_forceoff(self): vm = self.service.create_vm(self.spec()) self.backend.shutdown_stalls = True - with patch.object(self.service_module, "POWER_ACTION_TIMEOUT", 0): - with self.assertRaises(self.error_type) as raised: - self.service.action(vm["uuid"], "shutdown") - self.assertEqual(raised.exception.status, 502) - self.assertEqual( - raised.exception.error_code, - "power_action_result_uncertain", - ) + result = self.service.action(vm["uuid"], "shutdown") + self.assertEqual(result, { + "uuid": vm["uuid"], "action": "shutdown", "command_accepted": True, + }) self.assertTrue(self.backend.domains[0].isActive()) + self.assertEqual(self.backend.shutdown_calls, 1) + # A graceful request is not sufficient proof for destructive cleanup. + with self.assertRaises(self.error_type) as deletion: + self.service.delete_vm(vm["uuid"], True) + self.assertEqual(deletion.exception.status, 409) + self.assertTrue(self.backend.pools["images"].volumes) + forced = self.service.action(vm["uuid"], "poweroff") + self.assertTrue(forced["command_accepted"]) + self.assertFalse(self.backend.domains[0].isActive()) + self.assertEqual(self.backend.destroy_calls, 1) - def test_power_action_accepts_a_lost_response_after_target_state_commit(self): - vm = self.service.create_vm(self.spec()) - domain = self.backend.domains[0] - - def stop_then_fail(): - domain.running = False - domain.live_root = None - raise FakeLibvirtError("transport response lost") - - with patch.object(domain, "shutdown", side_effect=stop_then_fail): - result = self.service.action(vm["uuid"], "shutdown") - self.assertEqual(result["state"], "shutoff") - - def test_power_action_rechecks_start_after_response_loss(self): - vm = self.service.create_vm(self.spec()) - self.service.action(vm["uuid"], "shutdown") - domain = self.backend.domains[0] - original_create = domain.create - - def start_then_fail(): - original_create() - raise FakeLibvirtError("transport response lost") - - with patch.object(domain, "create", side_effect=start_then_fail): - result = self.service.action(vm["uuid"], "start") - self.assertEqual(result["state"], "running") - - def test_power_action_does_not_confirm_same_state_after_reboot_error(self): - for action in ("reboot", "reset"): + def test_power_receipts_do_not_depend_on_postreads_or_guest_agent(self): + for action in ("start", "shutdown", "reboot", "poweroff", "reset"): + with self.subTest(action=action): + service = self.service.__class__( + self.service.uri, "images", "iso", "default") + vm = service.create_vm(self.spec(name=f"receipt-{action}")) + domain = self.backend.domains[-1] + if action == "start": + domain.shutdown() + method = {"start": "create", "poweroff": "destroy"}.get(action, action) + original = getattr(domain, method) + invoked = False + original_active = domain.isActive + + def preflight_active_only(): + if invoked: + raise FakeLibvirtError("post-command state unavailable") + return original_active() + + def issue_command(*args): + nonlocal invoked + original(*args) + invoked = True + + with ( + patch.object(domain, method, side_effect=issue_command) as command, + patch.object(domain, "isActive", side_effect=preflight_active_only), + patch.object(domain, "info", side_effect=FakeLibvirtError( + "domain status unavailable")) as info, + patch.object(domain, "XMLDesc", side_effect=FakeLibvirtError( + "domain XML unavailable")) as xml, + patch.object(domain, "interfaceAddresses", side_effect=FakeLibvirtError( + "guest agent unavailable")) as guest_addresses, + patch.object(self.service_module.libvirt_qemu, + "qemuAgentCommand", side_effect=FakeLibvirtError( + "guest agent unavailable")) as guest_exec, + ): + result = service.action(vm["uuid"], action) + self.assertEqual(result, { + "uuid": vm["uuid"], "action": action, "command_accepted": True, + }) + command.assert_called_once_with(*((0,) if action == "reset" else ())) + info.assert_not_called() + xml.assert_not_called() + guest_addresses.assert_not_called() + guest_exec.assert_not_called() + + def test_power_action_does_not_infer_lost_receipt_from_current_state(self): + for action in ("start", "shutdown", "reboot", "poweroff", "reset"): with self.subTest(action=action): service = self.service.__class__( self.service.uri, "images", "iso", "default") - vm = service.create_vm(self.spec(name=f"power-{action}")) + vm = service.create_vm(self.spec(name=f"lost-{action}")) domain = self.backend.domains[-1] - if action == "reboot": - # The ISO installation fixture deliberately starts with - # on_reboot=destroy. Exercise the ordinary running-to- - # running reboot case so a lost response cannot be - # mistaken for a confirmed state change. - for root in (domain.live_root, domain.inactive_root): - node = root.find("on_reboot") - if node is None: - node = ET.SubElement(root, "on_reboot") - node.text = "restart" - - def fail_after_unknown_commit(*_args): + if action == "start": + domain.shutdown() + method = {"start": "create", "poweroff": "destroy"}.get(action, action) + original = getattr(domain, method) + + def commit_then_lose_response(*args): + original(*args) raise FakeLibvirtError("transport response lost") - with patch.object(domain, action, - side_effect=fail_after_unknown_commit): + with ( + patch.object(domain, method, side_effect=commit_then_lose_response) as command, + patch.object(domain, "info", side_effect=FakeLibvirtError( + "post-command state unavailable")) as info, + ): with self.assertRaises(self.error_type) as raised: service.action(vm["uuid"], action) self.assertEqual(raised.exception.status, 502) - self.assertEqual( - raised.exception.error_code, - "power_action_result_uncertain", - ) - self.assertTrue(domain.isActive()) - - def test_reboot_refuses_to_run_when_target_policy_cannot_be_read(self): - vm = self.service.create_vm(self.spec()) - domain = self.backend.domains[0] - with ( - patch.object(domain, "XMLDesc", side_effect=FakeLibvirtError( - "domain XML unavailable")), - patch.object(domain, "reboot") as reboot, - ): - with self.assertRaises(self.error_type) as raised: - self.service.action(vm["uuid"], "reboot") - self.assertEqual(raised.exception.status, 502) - self.assertEqual( - raised.exception.error_code, - "power_action_result_uncertain", - ) - self.assertTrue(domain.isActive()) - reboot.assert_not_called() + self.assertEqual(raised.exception.error_code, "power_action_result_uncertain") + command.assert_called_once_with(*((0,) if action == "reset" else ())) + info.assert_not_called() def test_service_suspend_and_resume_are_idempotent_target_states(self): vm = self.service.create_vm(self.spec()) diff --git a/engine/tests/test_resource_locks.py b/engine/tests/test_resource_locks.py index 1b3a3a96..8e6aab2c 100644 --- a/engine/tests/test_resource_locks.py +++ b/engine/tests/test_resource_locks.py @@ -242,3 +242,60 @@ def test_remote_uncertainty_releases_local_lock_but_retains_durable_resource(ser assert denied.value.status == 503 assert denied.value.error_code == "coordination_uncertain" assert module.RESOURCE_LOCKS.entry_count() == 0 + + +def test_accepted_shutdown_releases_durable_vm_claim_for_forceoff(monkeypatch): + from test_durable_jobs import MemoryCAS + from test_libvirt_service import VMServiceTests + from virsh_panel import coordination + + case = VMServiceTests() + case.setUp() + vm = case.service.create_vm(case.spec()) + coordinator = coordination.DurableCoordinator(MemoryCAS()) + monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator) + case.backend.shutdown_stalls = True + + receipt = case.service.action(vm["uuid"], "shutdown") + + assert receipt == {"uuid": vm["uuid"], "action": "shutdown", "command_accepted": True} + assert case.backend.domains[0].isActive() + assert coordinator.snapshot() == {} + assert case.service_module.RESOURCE_LOCKS.entry_count() == 0 + assert case.service.action(vm["uuid"], "poweroff")["command_accepted"] is True + assert not case.backend.domains[0].isActive() + assert case.backend.shutdown_calls == case.backend.destroy_calls == 1 + assert coordinator.snapshot() == {} + + +def test_power_response_loss_still_retains_durable_vm_claim(monkeypatch): + from test_durable_jobs import MemoryCAS + from test_libvirt_service import FakeLibvirtError, VMServiceTests + from virsh_panel import coordination + + case = VMServiceTests() + case.setUp() + vm = case.service.create_vm(case.spec()) + coordinator = coordination.DurableCoordinator(MemoryCAS()) + monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator) + domain = case.backend.domains[0] + shutdown = domain.shutdown + + def commit_then_lose_response(): + shutdown() + raise FakeLibvirtError("command response lost") + + monkeypatch.setattr(domain, "shutdown", commit_then_lose_response) + with pytest.raises(case.error_type) as lost: + case.service.action(vm["uuid"], "shutdown") + assert lost.value.error_code == "power_action_result_uncertain" + assert not domain.isActive() + claim = next(iter(coordinator.snapshot().values())) + assert claim["state"] == "uncertain" + assert case.service_module.RESOURCE_LOCKS.entry_count() == 0 + # A current shutoff state does not supply the missing command receipt. + starts_before = case.backend.domain_starts + with pytest.raises(case.error_type) as blocked: + case.service.action(vm["uuid"], "start") + assert blocked.value.error_code == "coordination_uncertain" + assert case.backend.domain_starts == starts_before diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index cc0a8ba3..1623cf41 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -6552,16 +6552,11 @@ def create_app( raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) vm = service.action(vm_id, body.action) if staged and body.action in {"poweroff", "start"}: - if body.action == "start": - if not vm.get("rescue_mode") or vm.get("rescue_staged"): - raise VMError( - "虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502) - rescue_record["status"] = "active" # A generic staged-rescue power action changes only the live - # QEMU state; the staged XML remains the same. Bind the - # durable marker to this authenticated action after libvirt - # confirms the target state so its private readback is not - # mistaken for a competing rescue operation. + # QEMU state; the staged XML remains the same. Bind its marker + # to the accepted command, without treating the receipt as + # proof of rescue boot. The rescue GET observes the live XML + # and promotes staged -> active independently. delegated = TRUSTED_ACTOR.get() if internal_mode else None action_operation_id = ( delegated.get("operation_id") diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 26546a02..aa2505e8 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -3382,8 +3382,8 @@ class EngineApplication: """Reconcile a lost synchronous power request by observation only. A power command is never replayed after its executor disappears. The - engine action itself verifies the postcondition before returning, and - this path covers the remaining transport/process-loss window by + engine action returns a command receipt without waiting for the guest. + This legacy path covers the remaining transport/process-loss window by reading the lightweight VM state endpoint. A transitional or mismatched state remains visible, while reboot/reset stay uncertain when the final state cannot prove that the edge-triggered command diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index dff96d39..d549ac13 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -113,13 +113,8 @@ NETWORK_CHANGE_SHUTDOWN_TIMEOUT = 120 NETWORK_CHANGE_SHUTDOWN_POLL_INTERVAL = 1 SERVICE_SUSPEND_SHUTDOWN_TIMEOUT = 30 SERVICE_SUSPEND_SHUTDOWN_POLL_INTERVAL = 0.5 -# libvirt's power APIs acknowledge a request before the guest necessarily -# reaches its final state. Keep the wait bounded so a guest that ignores an -# ACPI request becomes an explicit uncertain operation instead of a false -# success. The internal operation record can perform a later read-only -# reconciliation after a transport timeout. -POWER_ACTION_TIMEOUT = 60 -POWER_ACTION_POLL_INTERVAL = 0.5 +# A lost power command receipt remains uncertain even if a later read happens +# to show the requested state. Ordinary power commands do not await that state. POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE = "power_action_result_uncertain" VPC_GUEST_AGENT_TIMEOUT = 2 VPC_GUEST_AGENT_RESPONSE_MAX = 192 * 1024 @@ -6668,75 +6663,13 @@ class VMService: return self._create_iso_vm( conn, validated, requested_uuid, iso_pool, iso_volume) - @staticmethod - def _power_action_target_active(domain: Any, action: str) -> bool: - """Return the final active state expected for one power request. - - ``reboot`` is the one action whose final state is domain-configured: - an ``on_reboot=destroy`` policy intentionally leaves an ISO/install - domain shut off. If the policy cannot be read, refuse to issue the - edge-triggered request: accepting a running result would otherwise - hide an unknown target policy as a successful reboot. - """ - if action in {"start", "reset"}: - return True - if action in {"shutdown", "poweroff"}: - return False - if action == "reboot": - try: - root = ET.fromstring(domain.XMLDesc(0)) - except (libvirt.libvirtError, ET.ParseError, TypeError, ValueError) as exc: - raise VMError( - "无法确认重启后的目标状态,请查询实例状态后重试", - 502, - error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, - ) from exc - return root.findtext("on_reboot") != "destroy" - raise VMError("不支持的虚拟机操作") - - @classmethod - def _wait_for_power_action( - cls, domain: Any, *, action: str, target_active: bool, - timeout: float | None = None, - poll_interval: float | None = None, - ) -> None: - """Wait for and verify the bounded postcondition of a power action. + def action(self, uuid: str, action: str) -> dict[str, Any]: + """Return the libvirt command receipt, independently of guest readiness. - A successful libvirt call only means that the request was accepted. - Read ``info()[0]`` as well as ``isActive()`` so paused/blocked/crashed - domains cannot be mistaken for a running target. Reboot/reset cannot - prove that a complete guest reboot cycle occurred, but they can still - refuse a response that leaves the domain in a non-running state. + In particular, shutdown may return successfully while the guest stays + running. Status, XML and guest-agent reads must not turn that receipt + into an unknown mutation or prevent a later force-off request. """ - timeout = POWER_ACTION_TIMEOUT if timeout is None else timeout - poll_interval = ( - POWER_ACTION_POLL_INTERVAL - if poll_interval is None else poll_interval - ) - deadline = time.monotonic() + max(0, timeout) - while True: - try: - active = bool(domain.isActive()) - state = STATES.get(domain.info()[0], "unknown") - except (libvirt.libvirtError, IndexError, TypeError, ValueError) as exc: - raise VMError( - "虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交", - 502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, - ) from exc - reached = (active and state == "running") if target_active else ( - not active and state == "shutoff") - if reached: - return - if time.monotonic() >= deadline: - target = "运行" if target_active else "关机" - raise VMError( - f"虚拟机电源操作后未确认{target}状态,请查询实例状态;请勿重复提交", - 502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, - ) - if poll_interval > 0: - time.sleep(poll_interval) - - def action(self, uuid: str, action: str) -> dict[str, Any]: methods = { "start": "create", "shutdown": "shutdown", @@ -6746,6 +6679,7 @@ class VMService: } if action not in methods: raise VMError("不支持的虚拟机操作") + uuid = self._uuid(uuid) with ( self._operation_guard(uuid), self._connect(timeout=LIBVIRT_MUTATION_BUDGET) as conn, @@ -6756,33 +6690,19 @@ class VMService: raise VMError("虚拟机已经在运行", 409) if action != "start" and not active: raise VMError("虚拟机未运行", 409) - target_active = self._power_action_target_active(domain, action) try: if action == "reset": domain.reset(0) else: getattr(domain, methods[action])() except libvirt.libvirtError as exc: - # A lost libvirt response can still mean that the request - # committed. Verify the target before deciding whether to - # expose an uncertain result; never issue the action again. - # reset, and a normal reboot, deliberately keep the domain - # running. A running read after an exception is therefore - # indistinguishable from a command that never committed. - if action == "reset" or (action == "reboot" and target_active): - self._wait_for_power_action( - domain, action=action, target_active=target_active) - raise VMError( - "虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交", - 502, - error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, - ) from exc - self._wait_for_power_action( - domain, action=action, target_active=target_active) - else: - self._wait_for_power_action( - domain, action=action, target_active=target_active) - return self._vm_dict(domain) + # A current state cannot prove which command produced it. + # Preserve a missing receipt without replaying the command. + raise VMError( + "虚拟机电源操作结果无法确认,请核对原操作;请勿重复提交", + 502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, + ) from exc + return {"uuid": uuid, "action": action, "command_accepted": True} def service_suspend(self, uuid: str) -> dict[str, Any]: """Converge a persistent VM to the administratively stopped state. diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 3114a90a..73321828 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -169,7 +169,7 @@ final class CloudOperationErrorCatalog private const SAFE_RESULT_KEYS = [ 'phase', 'status', 'state', 'uuid', 'name', 'hostname', 'job_id', 'public_ip', 'public_ip_cidr', 'network_changed', 'skipped', - 'settings_changed', + 'settings_changed', 'command_accepted', 'action', 'snapshot_key', 'consistency', 'size_bytes', 'revision', 'policy_digest', 'created', 'restored', 'deleted', @@ -196,7 +196,9 @@ final class CloudOperationErrorCatalog public static function isDiagnosticKey(string $key): bool { $key = self::normalizeKey($key); - if ($key === '') { + if ($key === '' || $key === 'command_accepted') { + // This reviewed boolean is a receipt, never command text. Its + // value is still type-checked by safeResultScalar(). return false; } foreach (explode('_', $key) as $token) { @@ -707,7 +709,7 @@ final class CloudOperationErrorCatalog return in_array($key, ['network_changed', 'settings_changed', 'skipped', 'service_was_running', 'service_was_running_hint', 'autostart', 'managed', 'created', 'restored', 'deleted', - 'reinstall_committed', 'cleanup_pending'], true) + 'reinstall_committed', 'cleanup_pending', 'command_accepted'], true) ? $value : null; } if (is_int($value) || is_float($value)) { @@ -739,6 +741,12 @@ final class CloudOperationErrorCatalog return null; } $value = self::normalize($value); + if ($key === 'action') { + return CloudOperationCatalog::isPowerAction($value) ? $value : null; + } + if ($key === 'command_accepted') { + return null; + } if ($value === '' || mb_strlen($value) > 255) { return null; } diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 2ddb92fc..1e7f0c8d 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -393,8 +393,9 @@ final class CloudWorkerService $operation = ['status' => $operation['status'] ?? 'succeeded', 'result' => $operation]; } - // Every terminal VM mutation gets the same private-network - // readback gate. The engine's normal VM payload contains only a + // Mutations that own private-network state require its readback. + // A confirmed ordinary power receipt completes independently of + // guest/VPC readiness. The engine's normal VM payload contains only a // domain/XML observation; an attached VPC additionally requires // member identity, revision and digest evidence from the host // data-plane reconciler. A missing or stale observation is kept @@ -568,7 +569,8 @@ final class CloudWorkerService || in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) { return; } - if ($status === 'succeeded' && $job->instance_id !== null) { + if ($status === 'succeeded' && $job->instance_id !== null + && !self::confirmedPowerReceipt($job, $operation, $payload)) { $capturedPrivateExpectation = $payload['private_expectation'] ?? null; // VPC mutators lock VPC -> instance. Acquire the same order // before checking the captured target so detach/attach cannot @@ -1184,9 +1186,43 @@ final class CloudWorkerService return $status === 'uncertain' && $job->dispatched_at !== null; } + /** Bind a command receipt to the immutable ordinary-power request. */ + private static function confirmedPowerReceipt( + CloudJob $job, array $operation, array $payload + ): bool { + $result = $operation['result'] ?? null; + $action = self::payloadAction($payload); + if ((string) $job->kind !== 'action' + || !in_array($operation['status'] ?? null, + ['succeeded', 'completed', 'done'], true) + || $action === null || !CloudOperationCatalog::isPowerAction($action) + || ($payload['method'] ?? null) !== 'POST' + || !is_array($result) + || ($result['command_accepted'] ?? null) !== true + || ($result['action'] ?? null) !== $action + || !is_string($result['uuid'] ?? null) + || preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', + $result['uuid']) !== 1 + || !is_string($payload['resource'] ?? null) + || !hash_equals('vms/' . strtolower($result['uuid']) . '/actions', + strtolower($payload['resource']))) { + return false; + } + return !array_key_exists('operation_id', $operation) + || (is_string($operation['operation_id']) + && hash_equals((string) $job->operation_id, + $operation['operation_id'])); + } + private function applyPrivateLifecycleGate( CloudJob $job, array $operation, array $payload, array $actor ): array { + // libvirt shutdown/reboot acknowledge a request, not guest completion. + // The power command changes no VPC target and must not dispatch a + // network mutation or acquire a private-readiness dependency here. + if (self::confirmedPowerReceipt($job, $operation, $payload)) { + return $operation; + } $status = is_string($operation['status'] ?? null) ? strtolower((string) $operation['status']) : 'uncertain'; $kind = (string) $job->kind; @@ -1660,6 +1696,13 @@ final class CloudWorkerService $action = $kind === 'action' ? self::payloadAction($payload) : null; $normalized = CloudInstanceRuntimeService::normalizeState($state); if ($kind === 'action') { + // Receipt-only responses carry no observed runtime state. + // Shutdown has an explicit transition label; other actions + // retain their last observation until the normal poll runs. + if (($source['command_accepted'] ?? null) === true + && $normalized === 'unknown' && $action !== 'shutdown') { + return; + } // A successful graceful-shutdown job only confirms that the // request was accepted. libvirt may still report running (or // omit the state) until the guest completes ACPI shutdown. diff --git a/web/public/cloud/runtime.js b/web/public/cloud/runtime.js index 0d1531e1..7d0a27a3 100644 --- a/web/public/cloud/runtime.js +++ b/web/public/cloud/runtime.js @@ -6,8 +6,6 @@ const byId = (id) => document.getElementById(id); const state = { csrfToken: '', userId: '', username: '', admin: false, vms: [], offers: [], regions: [], lines: [], linesEnabled: false, sshKeys: [], templates: [], templateCatalogUnavailable: false, resources: null, selectedRegionId: '', selectedLineId: '', view: 'list', selectedVmId: null, detailTab: 'operations', metricsRange: '24h', metricsPayload: null, metricsRequest: null, metricsPollTimer: null, metricsResizeTimer: null, trafficDailyPayload: null, trafficDailyVmId: '', trafficDailyRequest: null, trafficDailyResizeTimer: null, trafficDailyLoadedAt: 0, credentialPassword: '', credentialVisible: false, credentialTimer: null, reinstallPassword: '', reinstallPasswordDismissed: false, rfb: null, consoleToken: 0, loadingVMs: false, selectedKeys: { create: '', reinstall: '' }, resourceController: null, mutationBusy: false, powerMutation: null, provisionIdempotency: '', provisionFingerprint: '', rescueActive: false, rescueStaged: false }; let vmStatusRequest = null; let vmStatusGeneration = 0; -const POWER_STATE_CONVERGENCE_INTERVAL_MS = 1500; -const POWER_STATE_CONVERGENCE_TIMEOUT_MS = 60000; // Keep the historical storage key so an interrupted deployment made just // before an online update can still reconcile to the same opening invoice. const PROVISION_INVOICE_STORAGE_PREFIX = 'virt-cloud:pending-order:'; @@ -374,43 +372,35 @@ function renderVmService(vm) { progress.classList.remove('is-exhausted'); } } -async function waitForShutdownState(vmId) { - const deadline = Date.now() + POWER_STATE_CONVERGENCE_TIMEOUT_MS; - while (Date.now() < deadline) { - const vm = await loadVmStatus(vmId, { live: true, force: true }); - if (vm && stateCategory(vm.state) === 'stopped') return true; - const remaining = deadline - Date.now(); - if (remaining <= 0) break; - await new Promise(resolve => setTimeout(resolve, - Math.min(POWER_STATE_CONVERGENCE_INTERVAL_MS, remaining))); - } - return false; -} async function powerAction(action, dangerous = false) { const vm = selectedVm(); if (!vm?.can_operate) return toast('实例服务已到期或正在执行策略,日常操作暂不可用。', 'error'); if (state.mutationBusy || state.powerMutation) return; if (dangerous && !confirm('此操作会强制断电,可能丢失未保存的数据。确认继续?')) return; + // A previous command's delayed observation must not overwrite the new one. + vmStatusGeneration += 1; + vmStatusRequest?.controller.abort(); + vmStatusRequest = null; const mutation = { vmId: vm.api_id, action }; state.powerMutation = mutation; renderVmDetail(vm); try { const result = await api(`${vmPath(vm)}/actions`, { method: 'POST', body: { action } }); - toast(`${result?.action_label || '电源'}命令发送成功`); - if (result.job_id) await pollJob('power', result.job_id); - if (action === 'shutdown') { - const stopped = await waitForShutdownState(vm.api_id); - if (stopped) toast('已确认实例完成关机。', 'success'); - else toast('关机任务已完成,但 60 秒内仍未确认实例停止;已恢复最后一次实时状态,请稍后刷新核查。', 'warning', '关机状态待确认'); - } else { - await loadVmStatus(vm.api_id, { live: true, force: true }); + if (result.job_id) { + toast(`${result?.action_label || '电源'}任务已提交,正在确认命令回执。`); + await pollJob('power', result.job_id); } + toast(action === 'shutdown' + ? '正常关机请求已确认;来宾可能仍在运行,实际电源状态将单独刷新。' + : `${result?.action_label || '电源'}命令已确认`, 'success'); } catch (error) { toast(error.message, 'error'); - void loadVmStatus(vm.api_id, { live: true, force: true }); } finally { if (state.powerMutation === mutation) state.powerMutation = null; if (selectedVm()) renderVmDetail(selectedVm()); + // Command completion releases controls; observed power state is independent. + // The regular detail poll continues even if this refresh fails or stays running. + void loadVmStatus(vm.api_id, { live: true, force: true }); } } async function deleteVm() { const vm = selectedVm(); if (!vm?.can_delete) return toast('当前账户无权删除该实例。', 'error'); if (state.mutationBusy || state.powerMutation) return; const confirmation = prompt(`永久删除“${vm.name}”及面板管理的虚拟磁盘,此操作不可恢复。请输入完整虚拟机名称确认:`); if (confirmation === null) return; if (confirmation !== vm.name) return toast('虚拟机名称不匹配,未执行删除。', 'error'); state.mutationBusy = true; setDetailBusy(true); try { const result = await api(vmPath(vm), { method: 'DELETE', body: { confirmation_name: vm.name, delete_disks: true } }); toast('删除命令发送成功'); if (result.job_id) await pollJob('delete', result.job_id); navigate(CLOUD_PATHS.instances); } catch (error) { toast(error.message, 'error'); } finally { state.mutationBusy = false; setDetailBusy(false); } } diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index d5ad9252..4531e86c 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026101203' scope='global'} +{assign var=venix_asset_version value='2026101301' scope='global'} diff --git a/web/tests/CloudInstanceReadPerformanceTest.php b/web/tests/CloudInstanceReadPerformanceTest.php index 210024de..00c31ce7 100644 --- a/web/tests/CloudInstanceReadPerformanceTest.php +++ b/web/tests/CloudInstanceReadPerformanceTest.php @@ -111,22 +111,21 @@ $assert(str_contains($detail, 'Initial, manual, BFCache and visible-page refreshes must bypass the runtime cache.'); $assert(str_contains($powerAction, 'state.powerMutation = mutation') && str_contains($powerAction, 'renderVmDetail(vm)') - && str_contains($powerAction, "if (action === 'shutdown')") - && str_contains($powerAction, 'await waitForShutdownState(vm.api_id)') - && str_contains($powerAction, "'关机状态待确认'") + && str_contains($powerAction, "await pollJob('power', result.job_id)") + && str_contains($powerAction, 'state.powerMutation = null') + && str_contains($powerAction, + 'void loadVmStatus(vm.api_id, { live: true, force: true })') + && str_contains($powerAction, 'vmStatusGeneration += 1') + && str_contains($powerAction, 'vmStatusRequest?.controller.abort()') && !str_contains($powerAction, 'state.mutationBusy = true') && !str_contains($powerAction, 'setDetailBusy(true)') - && str_contains($runtime, 'POWER_STATE_CONVERGENCE_INTERVAL_MS = 1500') - && str_contains($runtime, 'POWER_STATE_CONVERGENCE_TIMEOUT_MS = 60000') - && str_contains($runtime, - 'const deadline = Date.now() + POWER_STATE_CONVERGENCE_TIMEOUT_MS') - && str_contains($runtime, 'while (Date.now() < deadline)') + && !str_contains($runtime, 'waitForShutdownState') && str_contains($runtime, "label: '关机中'") && str_contains($runtime, 'const powerBusy = localPowerMutation(vm) !== null') && str_contains($runtime, "const consoleAvailable = running || transition === 'shutdown'") && str_contains($runtime, 'disabled: powerBusy || !running'), - 'Power actions must show shutdown convergence immediately without globally locking detail controls.'); + 'Power controls must wait only for the command result and refresh observed state independently.'); $assert(str_contains($instances, 'api(`/api/v1/vms/metadata?ids=${encodeURIComponent(ids.join(\',\'))}`)') && str_contains($instances, 'async function refreshMetadata()') diff --git a/web/tests/CloudPowerActionBehaviorTest.mjs b/web/tests/CloudPowerActionBehaviorTest.mjs new file mode 100644 index 00000000..c5e58e15 --- /dev/null +++ b/web/tests/CloudPowerActionBehaviorTest.mjs @@ -0,0 +1,168 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import vm from 'node:vm'; + +// Run the shipped command, job polling, status merging and button rendering. +// Only browser I/O is replaced so delayed reads cannot be hidden by a mock loader. +const moduleSource = await readFile(new URL('../public/cloud/runtime.js', import.meta.url), 'utf8'); +const source = moduleSource.replace(/^import[^\n]+\n/u, '') + .replace(/^export \{[\s\S]*?\};\s*$/mu, '') + + '\n;globalThis.__power = { state, selectedVm, renderVmDetail, powerAction, loadVmStatus };\n'; + +class Element { + constructor() { + this.children = []; this.textContent = ''; this.hidden = false; this.disabled = false; + this.dataset = {}; this.attributes = new Map(); this.listeners = new Map(); + this.style = {}; this.classList = { add() {}, remove() {}, toggle() {} }; + } + append(...children) { this.children.push(...children); } + replaceChildren(...children) { this.children = children; } + setAttribute(name, value) { this.attributes.set(name, String(value)); } + removeAttribute(name) { this.attributes.delete(name); } + addEventListener(name, callback) { this.listeners.set(name, callback); } + closest() { return this; } +} + +const flush = async () => { for (let turn = 0; turn < 30; turn += 1) await Promise.resolve(); }; + +function fixture() { + const elements = new Map(); + elements.set('cloud-page-error', new Element()); + const requests = []; + const notices = []; + const timers = []; + const context = { + document: { + visibilityState: 'visible', body: { dataset: { vmId: '17' } }, + getElementById(id) { + if (!elements.has(id)) elements.set(id, new Element()); + return elements.get(id); + }, + createElement() { return new Element(); }, + querySelectorAll() { return []; }, + addEventListener() {}, + }, + location: { protocol: 'https:', hostname: 'panel.test' }, + Headers, AbortController, URL, + confirm() { return true; }, + toast(...args) { notices.push(args); }, + setTimeout(callback, delay) { timers.push({ callback, delay }); return timers.length; }, + fetch(path, options = {}) { + return new Promise((resolve, reject) => requests.push({ path, ...options, resolve, reject, settled: false })); + }, + }; + vm.runInNewContext(source, context, { filename: 'runtime.js' }); + const runtime = context.__power; + const machine = { id: '17', api_id: '17', uuid: 'vm-uuid', name: 'test-vm', + can_operate: true, can_delete: true, state: 'running', state_fresh: true, + os_source: 'cloud_template' }; + Object.assign(runtime.state, { csrfToken: 'test-csrf', vms: [machine], selectedVmId: machine.uuid, view: 'detail' }); + runtime.renderVmDetail(machine); + return { + ...runtime, requests, notices, timers, elements, + button(label) { + const buttons = ['vm-detail-primary-actions', 'vm-detail-danger-actions'] + .flatMap(id => elements.get(id).children); + const button = buttons.find(item => item.textContent === label); + assert.ok(button, `Missing button: ${label}`); + return button; + }, + pending(path) { + const request = requests.find(item => !item.settled && item.path === path); + assert.ok(request, `Missing request: ${path}`); + return request; + }, + async respond(request, payload, status = 200) { + request.settled = true; + request.resolve({ status, ok: status >= 200 && status < 300, json: async () => payload }); + await flush(); + }, + }; +} + +async function accept(h, action, jobId = '41') { + const promise = h.powerAction(action, action === 'poweroff'); + await h.respond(h.pending('/api/v1/vms/17/actions'), { job_id: jobId, action_label: '电源' }); + return { promise, request: h.pending(`/api/v1/jobs/${jobId}`) }; +} + +// A request is not a command receipt: keep all power mutations serialized +// until the existing job has actually confirmed execution. +{ + const h = fixture(); + const pending = await accept(h, 'shutdown'); + assert.equal(h.button('强制断电').disabled, true); + await h.powerAction('poweroff', true); + assert.equal(h.requests.filter(item => item.method === 'POST').length, 1); + assert.equal(h.notices.some(([, kind]) => kind === 'success'), false); + await h.respond(pending.request, { status: 'succeeded' }); + await pending.promise; + assert.equal(h.state.powerMutation, null); + assert.equal(h.button('强制断电').disabled, false, 'A pending observation must not retain the power lock'); + assert.equal(h.selectedVm().state, 'running', 'The receipt does not invent a stopped state'); + assert.equal(h.timers.length, 0, 'No special shutdown convergence timer remains'); + + const observation = h.pending('/api/v1/vms/17/status?live=1'); + await h.respond(observation, { state: 'running', state_fresh: true }); + assert.equal(h.elements.get('vm-detail-state').textContent, '运行中'); + const forced = await accept(h, 'poweroff', '42'); + assert.equal(h.requests.filter(item => item.method === 'POST').length, 2); + await h.respond(forced.request, { status: 'succeeded' }); + await forced.promise; + await h.respond(h.pending('/api/v1/vms/17/status?live=1'), { state: 'shutoff', state_fresh: true }); + assert.equal(h.selectedVm().state, 'shutoff'); +} + +// A slow observation is invalidated when a newer action starts, even if fetch +// ignores abort and returns before the new command's post-completion refresh. +for (const lateError of [false, true]) { + const h = fixture(); + const shutdown = await accept(h, 'shutdown'); + await h.respond(shutdown.request, { status: 'succeeded' }); + await shutdown.promise; + const oldRead = h.pending('/api/v1/vms/17/status?live=1'); + const forced = await accept(h, 'poweroff', '42'); + assert.equal(oldRead.signal.aborted, true); + await h.respond(oldRead, lateError ? { message: 'stale observation error' } + : { state: 'shutoff', state_fresh: true }, lateError ? 502 : 200); + assert.equal(h.selectedVm().state, 'running', 'An old observation cannot update the newer command'); + assert.equal(h.elements.get('cloud-page-error').textContent, ''); + assert.equal(h.state.powerMutation.action, 'poweroff'); + await h.respond(forced.request, { status: 'succeeded' }); + await forced.promise; + await h.respond(h.pending('/api/v1/vms/17/status?live=1'), { state: 'shutoff', state_fresh: true }); + assert.equal(h.selectedVm().state, 'shutoff'); +} + +// Observation failure remains an observation error after a successful command; +// it neither changes the receipt nor disables a subsequent force-poweroff. +{ + const h = fixture(); + const shutdown = await accept(h, 'shutdown'); + await h.respond(shutdown.request, { status: 'succeeded' }); + await shutdown.promise; + await h.respond(h.pending('/api/v1/vms/17/status?live=1'), { message: '状态暂不可用' }, 502); + assert.equal(h.button('强制断电').disabled, false); + assert.equal(h.selectedVm().state, 'running'); + assert.equal(h.notices.filter(([, kind]) => kind === 'success').length, 1); + assert.equal(h.elements.get('cloud-page-error').textContent, '状态暂不可用'); +} + +// A genuinely unknown job is never presented as command success or replayed. +// Existing server-side admission still rejects a later explicit mutation. +{ + const h = fixture(); + const shutdown = await accept(h, 'shutdown'); + await h.respond(shutdown.request, { status: 'uncertain', error: '命令回执未知,请核对原操作' }); + await shutdown.promise; + assert.equal(h.notices.some(([, kind]) => kind === 'success'), false); + assert.equal(h.notices.at(-1)[1], 'error'); + assert.equal(h.requests.filter(item => item.method === 'POST').length, 1); + const forced = h.powerAction('poweroff', true); + await h.respond(h.pending('/api/v1/vms/17/actions'), { message: '已有待核对的任务' }, 409); + await forced; + assert.equal(h.notices.at(-1)[0], '已有待核对的任务'); + assert.equal(h.requests.filter(item => item.path.startsWith('/api/v1/jobs/')).length, 1); +} + +console.log('Cloud power command completion, independent observations and stale response checks passed.'); diff --git a/web/tests/CloudPowerReceiptWorkflowTest.php b/web/tests/CloudPowerReceiptWorkflowTest.php new file mode 100644 index 00000000..c6f4e1b4 --- /dev/null +++ b/web/tests/CloudPowerReceiptWorkflowTest.php @@ -0,0 +1,256 @@ +calls[] = [$host, $method, $resource, $body]; + if (!str_ends_with($resource, '/actions')) { + throw new CloudError('Private network/QGA unavailable', 503); + } + if ($this->afterCommand !== null) { ($this->afterCommand)(); } + return $this->response + ['operation_id' => $operation]; + } + } +} + +namespace { + require dirname(__DIR__) . '/vendor/autoload.php'; + + use App\Models\CloudInstance; + use App\Models\CloudInstanceRuntime; + use App\Models\CloudJob; + use App\Services\Cloud\CloudEngineClient; + use App\Services\Cloud\CloudError; + use App\Services\Cloud\CloudInstanceRuntimeService; + use App\Services\Cloud\CloudJobService; + use App\Services\Cloud\CloudOperationErrorCatalog; + use App\Services\Cloud\CloudSecrets; + use App\Services\Cloud\CloudVpcService; + use App\Services\Cloud\CloudWorkerService; + use Illuminate\Database\Capsule\Manager as Capsule; + use Illuminate\Database\Schema\Blueprint; + + $assert = static function (bool $condition, string $message): void { + if (!$condition) { throw new RuntimeException($message); } + }; + $capsule = new Capsule(); + $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema(); + $schema->create('user', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('enable')->default(1); + $t->boolean('is_admin')->default(false); + }); + $schema->create('cloud_instances', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('host_id'); $t->integer('user_id'); + $t->integer('order_id')->nullable(); $t->string('uuid'); $t->string('name'); + $t->string('status')->default('active'); + $t->string('service_status')->default('exempt'); + $t->string('network_status')->default('active'); + $t->integer('expires_at')->nullable(); $t->text('spec_json')->default('{}'); + $t->integer('updated_at')->default(0); + }); + $schema->create('cloud_jobs', static function (Blueprint $t): void { + $t->increments('id'); $t->string('operation_id')->unique(); + $t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable(); + $t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id'); + $t->string('kind'); $t->string('status'); $t->text('payload_cipher'); + $t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable(); + $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); + $t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at'); + $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); + }); + $schema->create('cloud_instance_runtime', static function (Blueprint $t): void { + $t->integer('instance_id')->primary(); $t->integer('host_id'); $t->string('uuid'); + $t->string('power_state'); $t->string('sync_state'); $t->string('last_error_code'); + foreach (['last_success_at', 'last_attempt_at', 'revision', 'created_at', 'updated_at'] as $key) { + $t->integer($key); + } + }); + $schema->create('cloud_vpcs', static function (Blueprint $t): void { + $t->increments('id'); $t->string('vpc_key'); $t->integer('desired_revision'); + $t->string('desired_digest'); $t->string('state'); $t->string('sync_state'); + }); + $schema->create('cloud_vpc_host_deployments', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('vpc_id'); $t->integer('host_id'); $t->string('state'); + }); + foreach (['cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) { + $schema->create($table, static function (Blueprint $t) use ($table): void { + $t->increments('id'); $t->integer('vpc_id'); $t->integer('subnet_id'); + $t->integer('instance_id'); $t->string('private_ip'); $t->string('state'); + $t->integer('desired_revision'); $t->string('desired_digest'); + $t->integer('observed_revision')->default(0); $t->string('observed_digest')->nullable(); + $t->string('last_error_code')->nullable(); $t->integer('updated_at')->default(0); + if ($table === 'cloud_vpc_members') { + $t->string('member_key'); $t->integer('host_id'); $t->integer('private_ip_id'); + $t->string('last_operation_id')->nullable(); + } else { + $t->integer('member_id'); + } + }); + } + $uuid = '11111111-1111-4111-8111-111111111110'; + Capsule::table('user')->insert(['id' => 1]); + Capsule::table('cloud_instances')->insert([ + 'id' => 10, 'host_id' => 7, 'user_id' => 1, 'uuid' => $uuid, 'name' => 'Power VM', + ]); + Capsule::table('cloud_vpcs')->insert([ + 'id' => 1, 'vpc_key' => str_repeat('a', 32), 'desired_revision' => 1, + 'desired_digest' => str_repeat('d', 64), 'state' => 'active', 'sync_state' => 'failed', + ]); + $member = ['id' => 1, 'vpc_id' => 1, 'subnet_id' => 1, 'instance_id' => 10, + 'private_ip' => '10.0.0.1', 'state' => 'uncertain', + 'desired_revision' => 1, 'desired_digest' => str_repeat('d', 64)]; + Capsule::table('cloud_vpc_members')->insert($member + [ + 'member_key' => str_repeat('b', 32), 'host_id' => 7, 'private_ip_id' => 1, + ]); + Capsule::table('cloud_vpc_private_ips')->insert($member + ['member_id' => 1]); + $actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, + 'can_manage' => false, 'session_id' => '']; + $engine = new CloudEngineClient(); + $worker = new CloudWorkerService($engine); + $enqueue = static fn (string $action): CloudJob => CloudJobService::enqueue( + CloudInstance::find(10), $actor, 'action', 'POST', '/actions', ['action' => $action] + ); + $expectation = CloudVpcService::instancePrivateExpectation(10); + $assert($expectation['required'] === true && $expectation['reason_code'] === null, + 'Fixture must have a valid attached private target despite failed readiness.'); + + // Each command can be followed immediately by another command even when + // the guest is broken and every private-readback endpoint would fail. + foreach (['shutdown', 'poweroff', 'start', 'reboot', 'reset'] as $action) { + $instance = CloudInstance::find(10); + $instance->network_status = $action === 'start' ? 'blocked_traffic' : 'active'; + $instance->save(); + $beforeState = $action === 'start' ? 'stopped' : 'running'; + CloudInstanceRuntimeService::markObserved(10, 7, $uuid, $beforeState); + $job = $enqueue($action); + $engine->response = ['status' => 'succeeded', 'result' => [ + 'uuid' => $uuid, 'action' => $action, 'command_accepted' => true, + ]]; + $count = count($engine->calls); + $assert($worker->tick(), 'Worker did not claim the power request.'); + $done = CloudJob::find($job->id); + $receipt = json_decode($done->result_json, true); + $assert($done->status === 'succeeded' && $done->lease_token === null + && (int) $done->lease_until === 0 && count($engine->calls) === $count + 1, + 'Confirmed ' . $action . ' waited for or dispatched private reconciliation.'); + $assert(($receipt['command_accepted'] ?? null) === true + && ($receipt['action'] ?? null) === $action && !isset($receipt['state']), + 'Power receipt was discarded or fabricated an observed power state.'); + $assert(CloudInstanceRuntime::find(10)->power_state + === ($action === 'shutdown' ? 'shutdown' : $beforeState), + 'Receipt-only command fabricated a running/stopped observation.'); + $assert(CloudInstance::find(10)->network_status === $instance->network_status + && Capsule::table('cloud_vpc_members')->value('state') === 'uncertain' + && Capsule::table('cloud_vpcs')->value('sync_state') === 'failed', + 'Ordinary power command changed independent network policy/readiness.'); + } + + // A changing private target does not invalidate an already accepted power + // command in finish()'s transaction either. + $job = $enqueue('shutdown'); + $engine->response = ['status' => 'succeeded', 'result' => [ + 'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true, + ]]; + $engine->afterCommand = static function (): void { + foreach (['cloud_vpcs', 'cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) { + Capsule::table($table)->update(['desired_revision' => 2]); + } + }; + $worker->tick(); + $assert(CloudJob::find($job->id)->status === 'succeeded', + 'Private revision change retroactively blocked confirmed power receipt.'); + $engine->afterCommand = null; + $next = $enqueue('poweroff'); + $assert($next->status === 'queued', 'Accepted shutdown still blocks force poweroff.'); + // Isolate the following rejection cases; no production cancellation code is used. + $next->status = 'cancelled'; $next->save(); + + $gate = new ReflectionMethod(CloudWorkerService::class, 'applyPrivateLifecycleGate'); + $confirmed = new ReflectionMethod(CloudWorkerService::class, 'confirmedPowerReceipt'); + $job = $enqueue('shutdown'); + $payload = CloudSecrets::decrypt($job->payload_cipher); + $success = ['operation_id' => $job->operation_id, 'status' => 'succeeded', 'result' => [ + 'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true, + ]]; + $assert($confirmed->invoke(null, $job, $success, $payload), 'Valid receipt not recognized.'); + $invalid = [ + array_replace_recursive($success, ['result' => ['uuid' => '22222222-2222-4222-8222-222222222222']]), + array_replace_recursive($success, ['result' => ['action' => 'poweroff']]), + array_replace_recursive($success, ['result' => ['command_accepted' => 'true']]), + array_replace_recursive($success, ['result' => ['command_accepted' => false]]), + array_replace($success, ['operation_id' => str_repeat('f', 32)]), + array_replace($success, ['status' => 'uncertain']), + array_replace($success, ['status' => 'running']), + array_replace($success, ['status' => 'failed']), + ]; + foreach ($invalid as $operation) { + $assert(!$confirmed->invoke(null, $job, $operation, $payload), + 'Mismatched, unconfirmed or nonterminal receipt bypassed the private gate.'); + } + foreach ([['method' => 'GET'], ['resource' => 'vms/' . $uuid . '/reinstall'], + ['body' => ['action' => 'delete']]] as $change) { + $assert(!$confirmed->invoke(null, $job, $success, array_replace($payload, $change)), + 'Receipt bypass was not bound to the immutable power request.'); + } + $safe = CloudOperationErrorCatalog::workerResult([ + 'command_accepted' => 'true', 'action' => 'arbitrary-host-message', + ]); + $assert($safe === [], 'Unvalidated power receipt fields reached stored/user results.'); + + // A lost power response is still uncertain, and the next write is rejected. + $engine->response = ['status' => 'uncertain', 'mutation_started' => true, + 'error_code' => 'libvirt_execution_uncertain']; + $worker->tick(); + $assert(CloudJob::find($job->id)->status === 'uncertain', + 'A truly uncertain write was upgraded to success.'); + try { + $enqueue('poweroff'); + throw new RuntimeException('Unknown write allowed a new power mutation.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 409, 'Unknown write had an unexpected admission result.'); + } + + // Unavailable private proof still gates destructive and policy mutations. + // Invoke the production gate directly to avoid manufacturing their finance + // ledgers; these are not ordinary action jobs even if a result carries a receipt. + foreach (['delete', 'expiry_delete', 'reinstall', 'network_change', + 'network_suspend', 'network_resume', 'service_suspend', 'service_resume'] as $kind) { + $other = clone $job; + $other->kind = $kind; + $count = count($engine->calls); + $result = $gate->invoke($worker, $other, $success, $payload, $actor); + $assert($result['status'] === 'uncertain' && count($engine->calls) > $count, + 'Power receipt weakened private readback for ' . $kind . '.'); + } + echo "Power command receipt workflow tests passed.\n"; +}