fix: confirm generic VM power action state

dev
admin 1 month ago
parent be5efa7033
commit 887072ac61

@ -2157,7 +2157,7 @@ def test_admin_created_vm_also_has_external_owner(boundary):
def test_power_action_has_persistent_idempotency(boundary):
client, engine, _, _, services = boundary
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41)
@ -2170,6 +2170,78 @@ def test_power_action_has_persistent_idempotency(boundary):
assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409
operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json()
assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False
saved = json.loads(store.get("operation", "reboot-operation"))
assert saved["target_action"] == "reboot"
def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
components = engine.registry.components["hk"]
components.state.users.assign_vm(
host_scope(components.manager.profile), VM_ID, USER["user_id"])
services["hk"].vm["state"] = "stopped"
operation = {
"operation_id": "orphan-power-state", "host_id": "hk", "user_id": 41,
"kind": "mutation", "resource": f"vms/{VM_ID}/actions",
"vm_uuid": VM_ID, "status": "uncertain", "result": None,
"safe_to_refund": False, "created_at": 1,
"actor_can_manage": True, "actor_is_admin": False,
"target_action": "shutdown",
}
store.set("operation", operation["operation_id"], json.dumps(operation))
recovered = call(
client, "GET", "/internal/v1/operations/orphan-power-state", USER,
).json()
assert recovered["status"] == "succeeded"
assert recovered["result"] == {
"uuid": VM_ID, "name": "alma-test", "state": "stopped",
}
assert ("get_vm_state", VM_ID) in services["hk"].calls
assert not any(row[0] == "action" for row in services["hk"].calls)
services["hk"].vm["state"] = "running"
mismatched = {**operation, "operation_id": "orphan-power-mismatch",
"target_action": "poweroff"}
store.set("operation", mismatched["operation_id"], json.dumps(mismatched))
failed = call(
client, "GET", "/internal/v1/operations/orphan-power-mismatch", USER,
).json()
assert failed["status"] == "failed"
assert "不会自动重放" in failed["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
for action in ("reboot", "reset"):
ambiguous = {
**operation,
"operation_id": f"orphan-power-{action}",
"target_action": action,
}
store.set("operation", ambiguous["operation_id"], json.dumps(ambiguous))
unresolved = call(
client, "GET",
f"/internal/v1/operations/{ambiguous['operation_id']}", USER,
).json()
assert unresolved["status"] == "uncertain"
assert unresolved["result"] is None
assert "不会自动重放" in unresolved["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
services["hk"].vm["state"] = "stopped"
reboot_off = {
**operation,
"operation_id": "orphan-power-reboot-off",
"target_action": "reboot",
}
store.set("operation", reboot_off["operation_id"], json.dumps(reboot_off))
unresolved_off = call(
client, "GET",
"/internal/v1/operations/orphan-power-reboot-off", USER,
).json()
assert unresolved_off["status"] == "uncertain"
assert unresolved_off["result"] is None
assert "不会自动重放" in unresolved_off["error"]
assert not any(row[0] == "action" for row in services["hk"].calls)
def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary):

@ -3332,6 +3332,96 @@ class VMServiceTests(unittest.TestCase):
self.assertFalse(self.backend.pools["images"].volumes)
self.assertFalse(self.backend.domains)
def test_power_action_does_not_report_graceful_shutdown_before_shutoff(self):
vm = self.service.create_vm(self.spec())
self.backend.shutdown_stalls = True
with patch.object(self.service_module, "POWER_ACTION_TIMEOUT", 0):
with self.assertRaises(self.error_type) as raised:
self.service.action(vm["uuid"], "shutdown")
self.assertEqual(raised.exception.status, 502)
self.assertEqual(
raised.exception.error_code,
"power_action_result_uncertain",
)
self.assertTrue(self.backend.domains[0].isActive())
def test_power_action_accepts_a_lost_response_after_target_state_commit(self):
vm = self.service.create_vm(self.spec())
domain = self.backend.domains[0]
def stop_then_fail():
domain.running = False
domain.live_root = None
raise FakeLibvirtError("transport response lost")
with patch.object(domain, "shutdown", side_effect=stop_then_fail):
result = self.service.action(vm["uuid"], "shutdown")
self.assertEqual(result["state"], "shutoff")
def test_power_action_rechecks_start_after_response_loss(self):
vm = self.service.create_vm(self.spec())
self.service.action(vm["uuid"], "shutdown")
domain = self.backend.domains[0]
original_create = domain.create
def start_then_fail():
original_create()
raise FakeLibvirtError("transport response lost")
with patch.object(domain, "create", side_effect=start_then_fail):
result = self.service.action(vm["uuid"], "start")
self.assertEqual(result["state"], "running")
def test_power_action_does_not_confirm_same_state_after_reboot_error(self):
for action in ("reboot", "reset"):
with self.subTest(action=action):
service = self.service.__class__(
self.service.uri, "images", "iso", "default")
vm = service.create_vm(self.spec(name=f"power-{action}"))
domain = self.backend.domains[-1]
if action == "reboot":
# The ISO installation fixture deliberately starts with
# on_reboot=destroy. Exercise the ordinary running-to-
# running reboot case so a lost response cannot be
# mistaken for a confirmed state change.
for root in (domain.live_root, domain.inactive_root):
node = root.find("on_reboot")
if node is None:
node = ET.SubElement(root, "on_reboot")
node.text = "restart"
def fail_after_unknown_commit(*_args):
raise FakeLibvirtError("transport response lost")
with patch.object(domain, action,
side_effect=fail_after_unknown_commit):
with self.assertRaises(self.error_type) as raised:
service.action(vm["uuid"], action)
self.assertEqual(raised.exception.status, 502)
self.assertEqual(
raised.exception.error_code,
"power_action_result_uncertain",
)
self.assertTrue(domain.isActive())
def test_reboot_refuses_to_run_when_target_policy_cannot_be_read(self):
vm = self.service.create_vm(self.spec())
domain = self.backend.domains[0]
with (
patch.object(domain, "XMLDesc", side_effect=FakeLibvirtError(
"domain XML unavailable")),
patch.object(domain, "reboot") as reboot,
):
with self.assertRaises(self.error_type) as raised:
self.service.action(vm["uuid"], "reboot")
self.assertEqual(raised.exception.status, 502)
self.assertEqual(
raised.exception.error_code,
"power_action_result_uncertain",
)
self.assertTrue(domain.isActive())
reboot.assert_not_called()
def test_service_suspend_and_resume_are_idempotent_target_states(self):
vm = self.service.create_vm(self.spec())
domain = self.backend.domains[0]

@ -92,6 +92,13 @@ DELETE_PUBLIC_ERROR_CODES = {
"vm_delete_firewall_cleanup_pending",
"vm_delete_volume_cleanup_pending",
}
POWER_ACTIONS = frozenset({"start", "shutdown", "reboot", "poweroff", "reset"})
POWER_ACTION_EDGE_ACTIONS = frozenset({"reboot", "reset"})
POWER_ACTION_TERMINAL_STATES = {
"start": frozenset({"running"}),
"shutdown": frozenset({"shutoff", "stopped"}),
"poweroff": frozenset({"shutoff", "stopped"}),
}
def _reinstall_committed(job: dict) -> bool:
@ -771,6 +778,8 @@ class EngineApplication:
or self._is_network_change_resource(
operation.get("resource", ""))
or self._is_firewall_resource(
operation.get("resource", ""))
or self._is_power_action_resource(
operation.get("resource", "")))):
operation = await self._mutation_operation(operation)
if operation["status"] == "succeeded":
@ -1750,7 +1759,9 @@ class EngineApplication:
# Even rejected signed provisioning requests have a durable outcome.
# A worker that loses this HTTP response can release its frozen balance
# only by querying this same operation, without replaying the request.
from .app import CreateCloudBody, CreateISOBody, FirewallPolicyBody
from .app import (
ActionBody, CreateCloudBody, CreateISOBody, FirewallPolicyBody,
)
from pydantic import ValidationError
create = resource == "vms"
value = {}
@ -1761,6 +1772,11 @@ class EngineApplication:
raise ValueError("body must be an object")
if create:
(CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value)
elif self._is_power_action_resource(resource):
# Persist only the reviewed action name. The original body
# may contain future/private fields and must never become a
# reconciliation instruction after a process restart.
value = ActionBody.model_validate(value).model_dump()
elif self._is_firewall_resource(resource):
value = FirewallPolicyBody.model_validate(value).model_dump()
from .libvirt_service import (
@ -1791,6 +1807,8 @@ class EngineApplication:
# UUID-derived resource candidates, while old in-flight records
# continue to use their historical display-name disk convention.
record["resource_naming"] = "uuid-v1"
if self._is_power_action_resource(resource) and not error:
record["target_action"] = value["action"]
if resource.endswith("/service-resume"):
# Store only the allowlisted target-state option. It is needed to
# replay the exact same convergence request after a lost response;
@ -1885,6 +1903,8 @@ class EngineApplication:
return await self._firewall_operation(record, components)
if self._is_network_change_resource(record.get("resource", "")):
return await self._network_change_operation(record, components)
if self._is_power_action_resource(record.get("resource", "")):
return await self._power_action_operation(record, components)
target_method = self._service_target_method(record.get("resource", ""))
if target_method is not None:
if record["operation_id"] in self.active_operations:
@ -2681,6 +2701,125 @@ class EngineApplication:
json.dumps(record))
return record
async def _power_action_operation(self, record, components):
"""Reconcile a lost synchronous power request by observation only.
A power command is never replayed after its executor disappears. The
engine action itself verifies the postcondition before returning, and
this path covers the remaining transport/process-loss window by
reading the lightweight VM state endpoint. A transitional or
mismatched state remains visible, while reboot/reset stay uncertain
when the final state cannot prove that the edge-triggered command
committed.
"""
if (record["operation_id"] in self.active_operations
or int(time.time()) - int(record.get("created_at", 0)) < 120):
return record
action = record.get("target_action")
if action not in POWER_ACTIONS:
# Rows written before target_action was introduced retain the old
# no-replay behavior; do not infer an action from arbitrary body
# data during recovery.
return await self._finish_orphan_mutation(record)
if (record.get("actor_can_manage") is not True
and record.get("actor_is_admin") is not True):
return await self._finish_orphan_mutation(record)
profile = await asyncio.to_thread(components.manager.current)
if profile is None or profile.get("key_locked"):
return record
if not record.get("actor_is_admin"):
try:
owns = await asyncio.to_thread(
components.state.users.owns_vm,
self._host_scope(profile), record["vm_uuid"],
record["user_id"],
)
except Exception:
return record
if not owns:
return await self._finish_orphan_mutation(record)
service = self._service(profile)
getter = getattr(service, "get_vm_state", None)
if not callable(getter):
# Compatibility with a short-lived older service implementation;
# get_vm still returns the same bounded state field, at a higher
# cost. Production VMService always has get_vm_state().
getter = getattr(service, "get_vm", None)
if not callable(getter):
return record
from .libvirt_service import VMError
try:
# Keep the observation, conclusion, and durable operation write in
# the same per-VM critical section as every other mutation. A new
# action must not race this recovery read and make an otherwise
# matching state stale immediately.
async with self._target_lock(record["host_id"], record["vm_uuid"]):
try:
observed = await asyncio.to_thread(
getter, record["vm_uuid"])
except Exception as exc:
if isinstance(exc, VMError) and exc.status == 404:
record.update(
status="failed", result=None,
error="虚拟机不存在",
finished_at=int(time.time()),
)
await asyncio.to_thread(
self.store.set, "operation", record["operation_id"],
json.dumps(record),
)
# A transport/read failure is not evidence of either
# outcome; leave the record available for a later
# read-only reconciliation.
return record
if not isinstance(observed, dict):
return record
if observed.get("uuid") != record["vm_uuid"]:
return record
state = observed.get("state")
if action in POWER_ACTION_EDGE_ACTIONS:
# reset always leaves the domain running, and a normal
# reboot usually does too. A reboot that leaves the
# domain shut off may be valid under on_reboot=destroy,
# but this durable row does not contain that XML policy.
# No observed state can prove either edge-triggered
# command committed, so keep it uncertain even when the
# domain is paused, crashed, or in transition.
record.update(
status="uncertain", result=None,
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
)
record.pop("finished_at", None)
elif state not in POWER_ACTION_TERMINAL_STATES[action]:
record.update(
status="failed", result=None,
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
finished_at=int(time.time()),
)
else:
result = {
key: observed[key]
for key in ("uuid", "name", "state")
if key in observed
}
record.update(
status="succeeded", result=result, error=None,
finished_at=int(time.time()), safe_to_refund=False,
)
record.pop("error_code", None)
await asyncio.to_thread(
self.store.set, "operation", record["operation_id"],
json.dumps(record))
return record
except Exception:
# A lock/store failure is not evidence of the VM's power state;
# leave the durable uncertainty available for a later pass.
LOG.exception(
"Power action reconciliation remains unresolved: op=%s",
record.get("operation_id"),
)
return record
async def _finish_orphan_mutation(self, record):
if (record["operation_id"] in self.active_operations
or int(time.time()) - int(record.get("created_at", 0)) < 120):
@ -2711,6 +2850,10 @@ class EngineApplication:
def _is_network_change_resource(resource: str) -> bool:
return re.fullmatch(rf"vms/{VM_ID}/network", resource) is not None
@staticmethod
def _is_power_action_resource(resource: str) -> bool:
return re.fullmatch(rf"vms/{VM_ID}/actions", resource) is not None
@staticmethod
def _is_firewall_resource(resource: str) -> bool:
return re.fullmatch(rf"vms/{VM_ID}/firewall", resource) is not None

@ -87,6 +87,14 @@ NETWORK_CHANGE_SHUTDOWN_TIMEOUT = 120
NETWORK_CHANGE_SHUTDOWN_POLL_INTERVAL = 1
SERVICE_SUSPEND_SHUTDOWN_TIMEOUT = 30
SERVICE_SUSPEND_SHUTDOWN_POLL_INTERVAL = 0.5
# libvirt's power APIs acknowledge a request before the guest necessarily
# reaches its final state. Keep the wait bounded so a guest that ignores an
# ACPI request becomes an explicit uncertain operation instead of a false
# success. The internal operation record can perform a later read-only
# reconciliation after a transport timeout.
POWER_ACTION_TIMEOUT = 60
POWER_ACTION_POLL_INTERVAL = 0.5
POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE = "power_action_result_uncertain"
SNAPSHOT_JOB_POLL_INTERVAL = 0.5
SNAPSHOT_JOB_TIMEOUT = 4 * 60 * 60
SNAPSHOT_ABORT_TIMEOUT = 30
@ -4049,6 +4057,74 @@ class VMService:
return self._create_iso_vm(
conn, validated, requested_uuid, iso_pool, iso_volume)
@staticmethod
def _power_action_target_active(domain: Any, action: str) -> bool:
"""Return the final active state expected for one power request.
``reboot`` is the one action whose final state is domain-configured:
an ``on_reboot=destroy`` policy intentionally leaves an ISO/install
domain shut off. If the policy cannot be read, refuse to issue the
edge-triggered request: accepting a running result would otherwise
hide an unknown target policy as a successful reboot.
"""
if action in {"start", "reset"}:
return True
if action in {"shutdown", "poweroff"}:
return False
if action == "reboot":
try:
root = ET.fromstring(domain.XMLDesc(0))
except (libvirt.libvirtError, ET.ParseError, TypeError, ValueError) as exc:
raise VMError(
"无法确认重启后的目标状态,请查询实例状态后重试",
502,
error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE,
) from exc
return root.findtext("on_reboot") != "destroy"
raise VMError("不支持的虚拟机操作")
@classmethod
def _wait_for_power_action(
cls, domain: Any, *, action: str, target_active: bool,
timeout: float | None = None,
poll_interval: float | None = None,
) -> None:
"""Wait for and verify the bounded postcondition of a power action.
A successful libvirt call only means that the request was accepted.
Read ``info()[0]`` as well as ``isActive()`` so paused/blocked/crashed
domains cannot be mistaken for a running target. Reboot/reset cannot
prove that a complete guest reboot cycle occurred, but they can still
refuse a response that leaves the domain in a non-running state.
"""
timeout = POWER_ACTION_TIMEOUT if timeout is None else timeout
poll_interval = (
POWER_ACTION_POLL_INTERVAL
if poll_interval is None else poll_interval
)
deadline = time.monotonic() + max(0, timeout)
while True:
try:
active = bool(domain.isActive())
state = STATES.get(domain.info()[0], "unknown")
except (libvirt.libvirtError, IndexError, TypeError, ValueError) as exc:
raise VMError(
"虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交",
502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE,
) from exc
reached = (active and state == "running") if target_active else (
not active and state == "shutoff")
if reached:
return
if time.monotonic() >= deadline:
target = "运行" if target_active else "关机"
raise VMError(
f"虚拟机电源操作后未确认{target}状态,请查询实例状态;请勿重复提交",
502, error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE,
)
if poll_interval > 0:
time.sleep(poll_interval)
def action(self, uuid: str, action: str) -> dict[str, Any]:
methods = {
"start": "create",
@ -4066,10 +4142,32 @@ class VMService:
raise VMError("虚拟机已经在运行", 409)
if action != "start" and not active:
raise VMError("虚拟机未运行", 409)
if action == "reset":
domain.reset(0)
target_active = self._power_action_target_active(domain, action)
try:
if action == "reset":
domain.reset(0)
else:
getattr(domain, methods[action])()
except libvirt.libvirtError as exc:
# A lost libvirt response can still mean that the request
# committed. Verify the target before deciding whether to
# expose an uncertain result; never issue the action again.
# reset, and a normal reboot, deliberately keep the domain
# running. A running read after an exception is therefore
# indistinguishable from a command that never committed.
if action == "reset" or (action == "reboot" and target_active):
self._wait_for_power_action(
domain, action=action, target_active=target_active)
raise VMError(
"虚拟机电源操作结果无法确认,请查询实例状态;请勿重复提交",
502,
error_code=POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE,
) from exc
self._wait_for_power_action(
domain, action=action, target_active=target_active)
else:
getattr(domain, methods[action])()
self._wait_for_power_action(
domain, action=action, target_active=target_active)
return self._vm_dict(domain)
def service_suspend(self, uuid: str) -> dict[str, Any]:

Loading…
Cancel
Save