diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index ca30bb9e..5510a8c4 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -2157,7 +2157,7 @@ def test_admin_created_vm_also_has_external_owner(boundary): def test_power_action_has_persistent_idempotency(boundary): - client, engine, _, _, services = boundary + client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41) @@ -2170,6 +2170,78 @@ def test_power_action_has_persistent_idempotency(boundary): assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409 operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json() assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False + saved = json.loads(store.get("operation", "reboot-operation")) + assert saved["target_action"] == "reboot" + + +def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary): + client, engine, store, _, services = boundary + call(client, "GET", "/internal/v1/hosts/hk/vms") + components = engine.registry.components["hk"] + components.state.users.assign_vm( + host_scope(components.manager.profile), VM_ID, USER["user_id"]) + services["hk"].vm["state"] = "stopped" + operation = { + "operation_id": "orphan-power-state", "host_id": "hk", "user_id": 41, + "kind": "mutation", "resource": f"vms/{VM_ID}/actions", + "vm_uuid": VM_ID, "status": "uncertain", "result": None, + "safe_to_refund": False, "created_at": 1, + "actor_can_manage": True, "actor_is_admin": False, + "target_action": "shutdown", + } + store.set("operation", operation["operation_id"], json.dumps(operation)) + recovered = call( + client, "GET", "/internal/v1/operations/orphan-power-state", USER, + ).json() + assert recovered["status"] == "succeeded" + assert recovered["result"] == { + "uuid": VM_ID, "name": "alma-test", "state": "stopped", + } + assert ("get_vm_state", VM_ID) in services["hk"].calls + assert not any(row[0] == "action" for row in services["hk"].calls) + + services["hk"].vm["state"] = "running" + mismatched = {**operation, "operation_id": "orphan-power-mismatch", + "target_action": "poweroff"} + store.set("operation", mismatched["operation_id"], json.dumps(mismatched)) + failed = call( + client, "GET", "/internal/v1/operations/orphan-power-mismatch", USER, + ).json() + assert failed["status"] == "failed" + assert "不会自动重放" in failed["error"] + assert not any(row[0] == "action" for row in services["hk"].calls) + + for action in ("reboot", "reset"): + ambiguous = { + **operation, + "operation_id": f"orphan-power-{action}", + "target_action": action, + } + store.set("operation", ambiguous["operation_id"], json.dumps(ambiguous)) + unresolved = call( + client, "GET", + f"/internal/v1/operations/{ambiguous['operation_id']}", USER, + ).json() + assert unresolved["status"] == "uncertain" + assert unresolved["result"] is None + assert "不会自动重放" in unresolved["error"] + assert not any(row[0] == "action" for row in services["hk"].calls) + + services["hk"].vm["state"] = "stopped" + reboot_off = { + **operation, + "operation_id": "orphan-power-reboot-off", + "target_action": "reboot", + } + store.set("operation", reboot_off["operation_id"], json.dumps(reboot_off)) + unresolved_off = call( + client, "GET", + "/internal/v1/operations/orphan-power-reboot-off", USER, + ).json() + assert unresolved_off["status"] == "uncertain" + assert unresolved_off["result"] is None + assert "不会自动重放" in unresolved_off["error"] + assert not any(row[0] == "action" for row in services["hk"].calls) def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary): diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 233935fd..4cc95341 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -3332,6 +3332,96 @@ class VMServiceTests(unittest.TestCase): self.assertFalse(self.backend.pools["images"].volumes) self.assertFalse(self.backend.domains) + def test_power_action_does_not_report_graceful_shutdown_before_shutoff(self): + vm = self.service.create_vm(self.spec()) + self.backend.shutdown_stalls = True + with patch.object(self.service_module, "POWER_ACTION_TIMEOUT", 0): + with self.assertRaises(self.error_type) as raised: + self.service.action(vm["uuid"], "shutdown") + self.assertEqual(raised.exception.status, 502) + self.assertEqual( + raised.exception.error_code, + "power_action_result_uncertain", + ) + self.assertTrue(self.backend.domains[0].isActive()) + + def test_power_action_accepts_a_lost_response_after_target_state_commit(self): + vm = self.service.create_vm(self.spec()) + domain = self.backend.domains[0] + + def stop_then_fail(): + domain.running = False + domain.live_root = None + raise FakeLibvirtError("transport response lost") + + with patch.object(domain, "shutdown", side_effect=stop_then_fail): + result = self.service.action(vm["uuid"], "shutdown") + self.assertEqual(result["state"], "shutoff") + + def test_power_action_rechecks_start_after_response_loss(self): + vm = self.service.create_vm(self.spec()) + self.service.action(vm["uuid"], "shutdown") + domain = self.backend.domains[0] + original_create = domain.create + + def start_then_fail(): + original_create() + raise FakeLibvirtError("transport response lost") + + with patch.object(domain, "create", side_effect=start_then_fail): + result = self.service.action(vm["uuid"], "start") + self.assertEqual(result["state"], "running") + + def test_power_action_does_not_confirm_same_state_after_reboot_error(self): + for action in ("reboot", "reset"): + with self.subTest(action=action): + service = self.service.__class__( + self.service.uri, "images", "iso", "default") + vm = service.create_vm(self.spec(name=f"power-{action}")) + domain = self.backend.domains[-1] + if action == "reboot": + # The ISO installation fixture deliberately starts with + # on_reboot=destroy. Exercise the ordinary running-to- + # running reboot case so a lost response cannot be + # mistaken for a confirmed state change. + for root in (domain.live_root, domain.inactive_root): + node = root.find("on_reboot") + if node is None: + node = ET.SubElement(root, "on_reboot") + node.text = "restart" + + def fail_after_unknown_commit(*_args): + raise FakeLibvirtError("transport response lost") + + with patch.object(domain, action, + side_effect=fail_after_unknown_commit): + with self.assertRaises(self.error_type) as raised: + service.action(vm["uuid"], action) + self.assertEqual(raised.exception.status, 502) + self.assertEqual( + raised.exception.error_code, + "power_action_result_uncertain", + ) + self.assertTrue(domain.isActive()) + + def test_reboot_refuses_to_run_when_target_policy_cannot_be_read(self): + vm = self.service.create_vm(self.spec()) + domain = self.backend.domains[0] + with ( + patch.object(domain, "XMLDesc", side_effect=FakeLibvirtError( + "domain XML unavailable")), + patch.object(domain, "reboot") as reboot, + ): + with self.assertRaises(self.error_type) as raised: + self.service.action(vm["uuid"], "reboot") + self.assertEqual(raised.exception.status, 502) + self.assertEqual( + raised.exception.error_code, + "power_action_result_uncertain", + ) + self.assertTrue(domain.isActive()) + reboot.assert_not_called() + def test_service_suspend_and_resume_are_idempotent_target_states(self): vm = self.service.create_vm(self.spec()) domain = self.backend.domains[0] diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 16927519..bb6e7981 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -92,6 +92,13 @@ DELETE_PUBLIC_ERROR_CODES = { "vm_delete_firewall_cleanup_pending", "vm_delete_volume_cleanup_pending", } +POWER_ACTIONS = frozenset({"start", "shutdown", "reboot", "poweroff", "reset"}) +POWER_ACTION_EDGE_ACTIONS = frozenset({"reboot", "reset"}) +POWER_ACTION_TERMINAL_STATES = { + "start": frozenset({"running"}), + "shutdown": frozenset({"shutoff", "stopped"}), + "poweroff": frozenset({"shutoff", "stopped"}), +} def _reinstall_committed(job: dict) -> bool: @@ -771,6 +778,8 @@ class EngineApplication: or self._is_network_change_resource( operation.get("resource", "")) or self._is_firewall_resource( + operation.get("resource", "")) + or self._is_power_action_resource( operation.get("resource", "")))): operation = await self._mutation_operation(operation) if operation["status"] == "succeeded": @@ -1750,7 +1759,9 @@ class EngineApplication: # Even rejected signed provisioning requests have a durable outcome. # A worker that loses this HTTP response can release its frozen balance # only by querying this same operation, without replaying the request. - from .app import CreateCloudBody, CreateISOBody, FirewallPolicyBody + from .app import ( + ActionBody, CreateCloudBody, CreateISOBody, FirewallPolicyBody, + ) from pydantic import ValidationError create = resource == "vms" value = {} @@ -1761,6 +1772,11 @@ class EngineApplication: raise ValueError("body must be an object") if create: (CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value) + elif self._is_power_action_resource(resource): + # Persist only the reviewed action name. The original body + # may contain future/private fields and must never become a + # reconciliation instruction after a process restart. + value = ActionBody.model_validate(value).model_dump() elif self._is_firewall_resource(resource): value = FirewallPolicyBody.model_validate(value).model_dump() from .libvirt_service import ( @@ -1791,6 +1807,8 @@ class EngineApplication: # UUID-derived resource candidates, while old in-flight records # continue to use their historical display-name disk convention. record["resource_naming"] = "uuid-v1" + if self._is_power_action_resource(resource) and not error: + record["target_action"] = value["action"] if resource.endswith("/service-resume"): # Store only the allowlisted target-state option. It is needed to # replay the exact same convergence request after a lost response; @@ -1885,6 +1903,8 @@ class EngineApplication: return await self._firewall_operation(record, components) if self._is_network_change_resource(record.get("resource", "")): return await self._network_change_operation(record, components) + if self._is_power_action_resource(record.get("resource", "")): + return await self._power_action_operation(record, components) target_method = self._service_target_method(record.get("resource", "")) if target_method is not None: if record["operation_id"] in self.active_operations: @@ -2681,6 +2701,125 @@ class EngineApplication: json.dumps(record)) return record + async def _power_action_operation(self, record, components): + """Reconcile a lost synchronous power request by observation only. + + A power command is never replayed after its executor disappears. The + engine action itself verifies the postcondition before returning, and + this path covers the remaining transport/process-loss window by + reading the lightweight VM state endpoint. A transitional or + mismatched state remains visible, while reboot/reset stay uncertain + when the final state cannot prove that the edge-triggered command + committed. + """ + if (record["operation_id"] in self.active_operations + or int(time.time()) - int(record.get("created_at", 0)) < 120): + return record + action = record.get("target_action") + if action not in POWER_ACTIONS: + # Rows written before target_action was introduced retain the old + # no-replay behavior; do not infer an action from arbitrary body + # data during recovery. + return await self._finish_orphan_mutation(record) + if (record.get("actor_can_manage") is not True + and record.get("actor_is_admin") is not True): + return await self._finish_orphan_mutation(record) + profile = await asyncio.to_thread(components.manager.current) + if profile is None or profile.get("key_locked"): + return record + if not record.get("actor_is_admin"): + try: + owns = await asyncio.to_thread( + components.state.users.owns_vm, + self._host_scope(profile), record["vm_uuid"], + record["user_id"], + ) + except Exception: + return record + if not owns: + return await self._finish_orphan_mutation(record) + service = self._service(profile) + getter = getattr(service, "get_vm_state", None) + if not callable(getter): + # Compatibility with a short-lived older service implementation; + # get_vm still returns the same bounded state field, at a higher + # cost. Production VMService always has get_vm_state(). + getter = getattr(service, "get_vm", None) + if not callable(getter): + return record + from .libvirt_service import VMError + try: + # Keep the observation, conclusion, and durable operation write in + # the same per-VM critical section as every other mutation. A new + # action must not race this recovery read and make an otherwise + # matching state stale immediately. + async with self._target_lock(record["host_id"], record["vm_uuid"]): + try: + observed = await asyncio.to_thread( + getter, record["vm_uuid"]) + except Exception as exc: + if isinstance(exc, VMError) and exc.status == 404: + record.update( + status="failed", result=None, + error="虚拟机不存在", + finished_at=int(time.time()), + ) + await asyncio.to_thread( + self.store.set, "operation", record["operation_id"], + json.dumps(record), + ) + # A transport/read failure is not evidence of either + # outcome; leave the record available for a later + # read-only reconciliation. + return record + if not isinstance(observed, dict): + return record + if observed.get("uuid") != record["vm_uuid"]: + return record + state = observed.get("state") + if action in POWER_ACTION_EDGE_ACTIONS: + # reset always leaves the domain running, and a normal + # reboot usually does too. A reboot that leaves the + # domain shut off may be valid under on_reboot=destroy, + # but this durable row does not contain that XML policy. + # No observed state can prove either edge-triggered + # command committed, so keep it uncertain even when the + # domain is paused, crashed, or in transition. + record.update( + status="uncertain", result=None, + error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态", + ) + record.pop("finished_at", None) + elif state not in POWER_ACTION_TERMINAL_STATES[action]: + record.update( + status="failed", result=None, + error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态", + finished_at=int(time.time()), + ) + else: + result = { + key: observed[key] + for key in ("uuid", "name", "state") + if key in observed + } + record.update( + status="succeeded", result=result, error=None, + finished_at=int(time.time()), safe_to_refund=False, + ) + record.pop("error_code", None) + await asyncio.to_thread( + self.store.set, "operation", record["operation_id"], + json.dumps(record)) + return record + except Exception: + # A lock/store failure is not evidence of the VM's power state; + # leave the durable uncertainty available for a later pass. + LOG.exception( + "Power action reconciliation remains unresolved: op=%s", + record.get("operation_id"), + ) + return record + async def _finish_orphan_mutation(self, record): if (record["operation_id"] in self.active_operations or int(time.time()) - int(record.get("created_at", 0)) < 120): @@ -2711,6 +2850,10 @@ class EngineApplication: def _is_network_change_resource(resource: str) -> bool: return re.fullmatch(rf"vms/{VM_ID}/network", resource) is not None + @staticmethod + def _is_power_action_resource(resource: str) -> bool: + return re.fullmatch(rf"vms/{VM_ID}/actions", resource) is not None + @staticmethod def _is_firewall_resource(resource: str) -> bool: return re.fullmatch(rf"vms/{VM_ID}/firewall", resource) is not None diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 853433f8..510215c3 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -87,6 +87,14 @@ NETWORK_CHANGE_SHUTDOWN_TIMEOUT = 120 NETWORK_CHANGE_SHUTDOWN_POLL_INTERVAL = 1 SERVICE_SUSPEND_SHUTDOWN_TIMEOUT = 30 SERVICE_SUSPEND_SHUTDOWN_POLL_INTERVAL = 0.5 +# libvirt's power APIs acknowledge a request before the guest necessarily +# reaches its final state. Keep the wait bounded so a guest that ignores an +# ACPI request becomes an explicit uncertain operation instead of a false +# success. The internal operation record can perform a later read-only +# reconciliation after a transport timeout. +POWER_ACTION_TIMEOUT = 60 +POWER_ACTION_POLL_INTERVAL = 0.5 +POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE = "power_action_result_uncertain" SNAPSHOT_JOB_POLL_INTERVAL = 0.5 SNAPSHOT_JOB_TIMEOUT = 4 * 60 * 60 SNAPSHOT_ABORT_TIMEOUT = 30 @@ -4049,6 +4057,74 @@ class VMService: return self._create_iso_vm( conn, validated, requested_uuid, iso_pool, iso_volume) + @staticmethod + def _power_action_target_active(domain: Any, action: str) -> bool: + """Return the final active state expected for one power request. + + ``reboot`` is the one action whose final state is domain-configured: + an ``on_reboot=destroy`` policy intentionally leaves an ISO/install + domain shut off. If the policy cannot be read, refuse to issue the + edge-triggered request: accepting a running result would otherwise + hide an unknown target policy as a successful reboot. + """ + if action in {"start", "reset"}: + return True + if action in {"shutdown", "poweroff"}: + return False + if action == "reboot": + try: + root = ET.fromstring(domain.XMLDesc(0)) + except (libvirt.libvirtError, ET.ParseError, TypeError, ValueError) as exc: + raise VMError( + "无法确认重启后的目标状态,请查询实例状态后重试", + 502, + error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, + ) from exc + return root.findtext("on_reboot") != "destroy" + raise VMError("不支持的虚拟机操作") + + @classmethod + def _wait_for_power_action( + cls, domain: Any, *, action: str, target_active: bool, + timeout: float | None = None, + poll_interval: float | None = None, + ) -> None: + """Wait for and verify the bounded postcondition of a power action. + + A successful libvirt call only means that the request was accepted. + Read ``info()[0]`` as well as ``isActive()`` so paused/blocked/crashed + domains cannot be mistaken for a running target. Reboot/reset cannot + prove that a complete guest reboot cycle occurred, but they can still + refuse a response that leaves the domain in a non-running state. + """ + timeout = POWER_ACTION_TIMEOUT if timeout is None else timeout + poll_interval = ( + POWER_ACTION_POLL_INTERVAL + if poll_interval is None else poll_interval + ) + deadline = time.monotonic() + max(0, timeout) + while True: + try: + active = bool(domain.isActive()) + state = STATES.get(domain.info()[0], "unknown") + except (libvirt.libvirtError, IndexError, TypeError, ValueError) as exc: + raise VMError( + "虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交", + 502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, + ) from exc + reached = (active and state == "running") if target_active else ( + not active and state == "shutoff") + if reached: + return + if time.monotonic() >= deadline: + target = "运行" if target_active else "关机" + raise VMError( + f"虚拟机电源操作后未确认{target}状态,请查询实例状态;请勿重复提交", + 502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, + ) + if poll_interval > 0: + time.sleep(poll_interval) + def action(self, uuid: str, action: str) -> dict[str, Any]: methods = { "start": "create", @@ -4066,10 +4142,32 @@ class VMService: raise VMError("虚拟机已经在运行", 409) if action != "start" and not active: raise VMError("虚拟机未运行", 409) - if action == "reset": - domain.reset(0) + target_active = self._power_action_target_active(domain, action) + try: + if action == "reset": + domain.reset(0) + else: + getattr(domain, methods[action])() + except libvirt.libvirtError as exc: + # A lost libvirt response can still mean that the request + # committed. Verify the target before deciding whether to + # expose an uncertain result; never issue the action again. + # reset, and a normal reboot, deliberately keep the domain + # running. A running read after an exception is therefore + # indistinguishable from a command that never committed. + if action == "reset" or (action == "reboot" and target_active): + self._wait_for_power_action( + domain, action=action, target_active=target_active) + raise VMError( + "虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交", + 502, + error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE, + ) from exc + self._wait_for_power_action( + domain, action=action, target_active=target_active) else: - getattr(domain, methods[action])() + self._wait_for_power_action( + domain, action=action, target_active=target_active) return self._vm_dict(domain) def service_suspend(self, uuid: str) -> dict[str, Any]: