|
|
|
|
@ -92,6 +92,13 @@ DELETE_PUBLIC_ERROR_CODES = {
|
|
|
|
|
"vm_delete_firewall_cleanup_pending",
|
|
|
|
|
"vm_delete_volume_cleanup_pending",
|
|
|
|
|
}
|
|
|
|
|
POWER_ACTIONS = frozenset({"start", "shutdown", "reboot", "poweroff", "reset"})
|
|
|
|
|
POWER_ACTION_EDGE_ACTIONS = frozenset({"reboot", "reset"})
|
|
|
|
|
POWER_ACTION_TERMINAL_STATES = {
|
|
|
|
|
"start": frozenset({"running"}),
|
|
|
|
|
"shutdown": frozenset({"shutoff", "stopped"}),
|
|
|
|
|
"poweroff": frozenset({"shutoff", "stopped"}),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _reinstall_committed(job: dict) -> bool:
|
|
|
|
|
@ -771,6 +778,8 @@ class EngineApplication:
|
|
|
|
|
or self._is_network_change_resource(
|
|
|
|
|
operation.get("resource", ""))
|
|
|
|
|
or self._is_firewall_resource(
|
|
|
|
|
operation.get("resource", ""))
|
|
|
|
|
or self._is_power_action_resource(
|
|
|
|
|
operation.get("resource", "")))):
|
|
|
|
|
operation = await self._mutation_operation(operation)
|
|
|
|
|
if operation["status"] == "succeeded":
|
|
|
|
|
@ -1750,7 +1759,9 @@ class EngineApplication:
|
|
|
|
|
# Even rejected signed provisioning requests have a durable outcome.
|
|
|
|
|
# A worker that loses this HTTP response can release its frozen balance
|
|
|
|
|
# only by querying this same operation, without replaying the request.
|
|
|
|
|
from .app import CreateCloudBody, CreateISOBody, FirewallPolicyBody
|
|
|
|
|
from .app import (
|
|
|
|
|
ActionBody, CreateCloudBody, CreateISOBody, FirewallPolicyBody,
|
|
|
|
|
)
|
|
|
|
|
from pydantic import ValidationError
|
|
|
|
|
create = resource == "vms"
|
|
|
|
|
value = {}
|
|
|
|
|
@ -1761,6 +1772,11 @@ class EngineApplication:
|
|
|
|
|
raise ValueError("body must be an object")
|
|
|
|
|
if create:
|
|
|
|
|
(CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value)
|
|
|
|
|
elif self._is_power_action_resource(resource):
|
|
|
|
|
# Persist only the reviewed action name. The original body
|
|
|
|
|
# may contain future/private fields and must never become a
|
|
|
|
|
# reconciliation instruction after a process restart.
|
|
|
|
|
value = ActionBody.model_validate(value).model_dump()
|
|
|
|
|
elif self._is_firewall_resource(resource):
|
|
|
|
|
value = FirewallPolicyBody.model_validate(value).model_dump()
|
|
|
|
|
from .libvirt_service import (
|
|
|
|
|
@ -1791,6 +1807,8 @@ class EngineApplication:
|
|
|
|
|
# UUID-derived resource candidates, while old in-flight records
|
|
|
|
|
# continue to use their historical display-name disk convention.
|
|
|
|
|
record["resource_naming"] = "uuid-v1"
|
|
|
|
|
if self._is_power_action_resource(resource) and not error:
|
|
|
|
|
record["target_action"] = value["action"]
|
|
|
|
|
if resource.endswith("/service-resume"):
|
|
|
|
|
# Store only the allowlisted target-state option. It is needed to
|
|
|
|
|
# replay the exact same convergence request after a lost response;
|
|
|
|
|
@ -1885,6 +1903,8 @@ class EngineApplication:
|
|
|
|
|
return await self._firewall_operation(record, components)
|
|
|
|
|
if self._is_network_change_resource(record.get("resource", "")):
|
|
|
|
|
return await self._network_change_operation(record, components)
|
|
|
|
|
if self._is_power_action_resource(record.get("resource", "")):
|
|
|
|
|
return await self._power_action_operation(record, components)
|
|
|
|
|
target_method = self._service_target_method(record.get("resource", ""))
|
|
|
|
|
if target_method is not None:
|
|
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
|
|
@ -2681,6 +2701,125 @@ class EngineApplication:
|
|
|
|
|
json.dumps(record))
|
|
|
|
|
return record
|
|
|
|
|
|
|
|
|
|
async def _power_action_operation(self, record, components):
|
|
|
|
|
"""Reconcile a lost synchronous power request by observation only.
|
|
|
|
|
|
|
|
|
|
A power command is never replayed after its executor disappears. The
|
|
|
|
|
engine action itself verifies the postcondition before returning, and
|
|
|
|
|
this path covers the remaining transport/process-loss window by
|
|
|
|
|
reading the lightweight VM state endpoint. A transitional or
|
|
|
|
|
mismatched state remains visible, while reboot/reset stay uncertain
|
|
|
|
|
when the final state cannot prove that the edge-triggered command
|
|
|
|
|
committed.
|
|
|
|
|
"""
|
|
|
|
|
if (record["operation_id"] in self.active_operations
|
|
|
|
|
or int(time.time()) - int(record.get("created_at", 0)) < 120):
|
|
|
|
|
return record
|
|
|
|
|
action = record.get("target_action")
|
|
|
|
|
if action not in POWER_ACTIONS:
|
|
|
|
|
# Rows written before target_action was introduced retain the old
|
|
|
|
|
# no-replay behavior; do not infer an action from arbitrary body
|
|
|
|
|
# data during recovery.
|
|
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
|
|
if (record.get("actor_can_manage") is not True
|
|
|
|
|
and record.get("actor_is_admin") is not True):
|
|
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
|
|
return record
|
|
|
|
|
if not record.get("actor_is_admin"):
|
|
|
|
|
try:
|
|
|
|
|
owns = await asyncio.to_thread(
|
|
|
|
|
components.state.users.owns_vm,
|
|
|
|
|
self._host_scope(profile), record["vm_uuid"],
|
|
|
|
|
record["user_id"],
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
return record
|
|
|
|
|
if not owns:
|
|
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
|
|
service = self._service(profile)
|
|
|
|
|
getter = getattr(service, "get_vm_state", None)
|
|
|
|
|
if not callable(getter):
|
|
|
|
|
# Compatibility with a short-lived older service implementation;
|
|
|
|
|
# get_vm still returns the same bounded state field, at a higher
|
|
|
|
|
# cost. Production VMService always has get_vm_state().
|
|
|
|
|
getter = getattr(service, "get_vm", None)
|
|
|
|
|
if not callable(getter):
|
|
|
|
|
return record
|
|
|
|
|
from .libvirt_service import VMError
|
|
|
|
|
try:
|
|
|
|
|
# Keep the observation, conclusion, and durable operation write in
|
|
|
|
|
# the same per-VM critical section as every other mutation. A new
|
|
|
|
|
# action must not race this recovery read and make an otherwise
|
|
|
|
|
# matching state stale immediately.
|
|
|
|
|
async with self._target_lock(record["host_id"], record["vm_uuid"]):
|
|
|
|
|
try:
|
|
|
|
|
observed = await asyncio.to_thread(
|
|
|
|
|
getter, record["vm_uuid"])
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
if isinstance(exc, VMError) and exc.status == 404:
|
|
|
|
|
record.update(
|
|
|
|
|
status="failed", result=None,
|
|
|
|
|
error="虚拟机不存在",
|
|
|
|
|
finished_at=int(time.time()),
|
|
|
|
|
)
|
|
|
|
|
await asyncio.to_thread(
|
|
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
|
|
json.dumps(record),
|
|
|
|
|
)
|
|
|
|
|
# A transport/read failure is not evidence of either
|
|
|
|
|
# outcome; leave the record available for a later
|
|
|
|
|
# read-only reconciliation.
|
|
|
|
|
return record
|
|
|
|
|
if not isinstance(observed, dict):
|
|
|
|
|
return record
|
|
|
|
|
if observed.get("uuid") != record["vm_uuid"]:
|
|
|
|
|
return record
|
|
|
|
|
state = observed.get("state")
|
|
|
|
|
if action in POWER_ACTION_EDGE_ACTIONS:
|
|
|
|
|
# reset always leaves the domain running, and a normal
|
|
|
|
|
# reboot usually does too. A reboot that leaves the
|
|
|
|
|
# domain shut off may be valid under on_reboot=destroy,
|
|
|
|
|
# but this durable row does not contain that XML policy.
|
|
|
|
|
# No observed state can prove either edge-triggered
|
|
|
|
|
# command committed, so keep it uncertain even when the
|
|
|
|
|
# domain is paused, crashed, or in transition.
|
|
|
|
|
record.update(
|
|
|
|
|
status="uncertain", result=None,
|
|
|
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
|
|
|
)
|
|
|
|
|
record.pop("finished_at", None)
|
|
|
|
|
elif state not in POWER_ACTION_TERMINAL_STATES[action]:
|
|
|
|
|
record.update(
|
|
|
|
|
status="failed", result=None,
|
|
|
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
|
|
|
finished_at=int(time.time()),
|
|
|
|
|
)
|
|
|
|
|
else:
|
|
|
|
|
result = {
|
|
|
|
|
key: observed[key]
|
|
|
|
|
for key in ("uuid", "name", "state")
|
|
|
|
|
if key in observed
|
|
|
|
|
}
|
|
|
|
|
record.update(
|
|
|
|
|
status="succeeded", result=result, error=None,
|
|
|
|
|
finished_at=int(time.time()), safe_to_refund=False,
|
|
|
|
|
)
|
|
|
|
|
record.pop("error_code", None)
|
|
|
|
|
await asyncio.to_thread(
|
|
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
|
|
json.dumps(record))
|
|
|
|
|
return record
|
|
|
|
|
except Exception:
|
|
|
|
|
# A lock/store failure is not evidence of the VM's power state;
|
|
|
|
|
# leave the durable uncertainty available for a later pass.
|
|
|
|
|
LOG.exception(
|
|
|
|
|
"Power action reconciliation remains unresolved: op=%s",
|
|
|
|
|
record.get("operation_id"),
|
|
|
|
|
)
|
|
|
|
|
return record
|
|
|
|
|
|
|
|
|
|
async def _finish_orphan_mutation(self, record):
|
|
|
|
|
if (record["operation_id"] in self.active_operations
|
|
|
|
|
or int(time.time()) - int(record.get("created_at", 0)) < 120):
|
|
|
|
|
@ -2711,6 +2850,10 @@ class EngineApplication:
|
|
|
|
|
def _is_network_change_resource(resource: str) -> bool:
|
|
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/network", resource) is not None
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def _is_power_action_resource(resource: str) -> bool:
|
|
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/actions", resource) is not None
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def _is_firewall_resource(resource: str) -> bool:
|
|
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/firewall", resource) is not None
|
|
|
|
|
|