"""Private PHP -> libvirt boundary. The legacy browser app is never exposed. Only allowlisted VM/host/template resources are dispatched to per-host services. The public console endpoint accepts a short-lived, one-use ticket instead of a PHP cookie, and rechecks the PHP session while the console remains connected. """ from __future__ import annotations import asyncio import errno import hashlib import hmac import inspect import json import logging import os import re import secrets import stat import subprocess import tempfile import threading import time import uuid from contextlib import AsyncExitStack, suppress from pathlib import Path from urllib.parse import parse_qsl, urlsplit from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLRequest, build_opener) from .cloud_templates import ( MAX_DOWNLOAD_SECONDS, MAX_IMAGE_BYTES, TemplateError, template_volume_name, validate_template_metadata, validate_template_name, verify_qcow2_image, ) from .cloud_template_runtime import ( CACHE_TOMBSTONE_KIND, MIN_DOWNLOAD_BYTES, CloudTemplateRuntime, ) from .engine_context import (EngineComponents, HostStore, LogStore, MainStore, TRUSTED_ACTOR, actor_session) from .engine_security import (EngineAuthError, authenticate, decode_secret, signed_headers) from .updater import UpdateError, Updater from .user_isos import UserISORuntime LOG = logging.getLogger(__name__) HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") OP_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z") VM_ID = r"[0-9a-fA-F-]{36}" JSON_LIMIT = 1024 * 1024 CLOUD_UPLOAD_IDLE_SECONDS = 60 CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024 CACHE_CLEANUP_RETRY_SECONDS = 30.0 DELETE_CLEANUP_SNAPSHOT_LIMIT = 64 SNAPSHOT_PUBLIC_ERROR_CODES = { "snapshot_guest_agent_unavailable", "live_snapshot_unsupported", "snapshot_pool_unsupported", "snapshot_result_uncertain", } FIREWALL_PUBLIC_ERROR_CODES = { "firewall_capability_missing", "firewall_rule_invalid", "firewall_apply_failed", "firewall_result_uncertain", "firewall_drift_detected", "firewall_cleanup_pending", } REINSTALL_PUBLIC_ERROR_CODES = { "reinstall_cleanup_pending", "reinstall_result_uncertain", } # Compatibility with reinstall rows written before the explicit commit marker # was introduced. These phases are only emitted after the replacement domain # has been defined/started or while its post-commit result is being saved. REINSTALL_COMMITTED_PHASES = frozenset({ "cleaning_up", "cleanup_pending", "result_uncertain", "completed", "重装已完成,但凭据或任务记录保存失败", "重装完成,但旧系统盘清理待核对", "重装结果待核对", }) DELETE_PUBLIC_ERROR_CODES = { "public_ip_claim_cleanup_pending", "vm_delete_snapshot_cleanup_pending", "vm_delete_firewall_cleanup_pending", "vm_delete_volume_cleanup_pending", } POWER_ACTIONS = frozenset({"start", "shutdown", "reboot", "poweroff", "reset"}) POWER_ACTION_EDGE_ACTIONS = frozenset({"reboot", "reset"}) POWER_ACTION_TERMINAL_STATES = { "start": frozenset({"running"}), "shutdown": frozenset({"shutoff", "stopped"}), "poweroff": frozenset({"shutoff", "stopped"}), } def _reinstall_committed(job: dict) -> bool: if job.get("reinstall_committed") is True: return True if job.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES: return True phase = job.get("phase") if isinstance(phase, str) and phase.strip() in REINSTALL_COMMITTED_PHASES: return True if job.get("cleanup_pending") is True: return True warning = job.get("warning") return isinstance(warning, str) and bool(warning.strip()) def _reinstall_cleanup_pending(job: dict) -> bool: phase = job.get("phase") if isinstance(phase, str): phase = phase.strip() if job.get("error_code") == "reinstall_cleanup_pending": return True if job.get("cleanup_pending") is True: return True if phase in {"cleanup_pending", "重装完成,但旧系统盘清理待核对"}: return True warning = job.get("warning") return isinstance(warning, str) and warning.strip().startswith("旧存储卷") READ_PATHS = ( r"host", r"host/network-pools", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms", r"vm-states", r"vm-metadata", r"user-isos/[0-9a-f]{32}", rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue|" r"/firewall|/snapshots(?:/[0-9a-f]{32})?)?", r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+", ) WRITE_PATHS = { "POST": (r"host/(?:scan|unlock|test)", r"vm-templates/(?:import|upload)", r"rescue-templates", r"user-isos/import", r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|network|settings|rescue|console-ticket)", rf"vms/{VM_ID}/snapshots(?:/[0-9a-f]{{32}}/restore)?", rf"vms/{VM_ID}/(?:service|network)-(?:suspend|resume)", rf"vms/{VM_ID}/rescue/(?:exit|force-start)", r"reinstall-jobs/[A-Za-z0-9_-]+/ack-password"), "PUT": (r"host", r"host/network-pools/[0-9a-f]{32}", rf"vms/{VM_ID}/firewall"), "PATCH": (r"vm-templates/[A-Za-z0-9_-]+",), "DELETE": (rf"vms/{VM_ID}(?:/snapshots/[0-9a-f]{{32}})?", r"vm-templates/[A-Za-z0-9_-]+", r"rescue-templates/[A-Za-z0-9_-]+", r"user-isos/[0-9a-f]{32}"), } class _CloudUploadWriter: """Own a temporary upload descriptor used only from worker threads.""" def __init__(self, directory: Path) -> None: descriptor, raw_path = tempfile.mkstemp( prefix=".cloud-image-upload-", suffix=".qcow2", dir=directory, ) try: os.fchmod(descriptor, 0o600) except BaseException: os.close(descriptor) Path(raw_path).unlink(missing_ok=True) raise self.path = Path(raw_path) self._descriptor = descriptor self._lock = threading.Lock() def write(self, chunk: memoryview) -> None: """Write the complete bounded chunk before accepting another one.""" with self._lock: if self._descriptor < 0: raise OSError(errno.EBADF, "cloud upload is closed") remaining = chunk while remaining: written = os.write(self._descriptor, remaining) if written <= 0: raise OSError(errno.EIO, "cloud upload write made no progress") remaining = remaining[written:] def sync_and_close(self) -> None: """Durably flush and close the descriptor before image validation.""" with self._lock: if self._descriptor < 0: return descriptor = self._descriptor try: os.fsync(descriptor) finally: try: os.close(descriptor) finally: self._descriptor = -1 def close(self) -> None: """Close after an interrupted request, waiting for an in-flight write.""" with self._lock: if self._descriptor < 0: return descriptor = self._descriptor try: os.close(descriptor) finally: self._descriptor = -1 class NoRedirects(HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): raise ValueError("Session introspection redirects are not allowed") class UpdateServiceUncertain(UpdateError): """The worker could not be confirmed stopped, so its job must stay active.""" def private_config(path: Path) -> dict: info = path.lstat() if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 or info.st_uid != os.geteuid(): raise ValueError("引擎配置必须为运行用户拥有的普通 0600 文件") descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) try: raw = os.read(descriptor, JSON_LIMIT + 1) finally: os.close(descriptor) if len(raw) > JSON_LIMIT: raise ValueError("引擎配置过大") value = json.loads(raw) if not isinstance(value, dict): raise ValueError("引擎配置无效") return value def master_key(path: Path) -> bytes: from .database_bootstrap import DatabaseBootstrap if not path.is_absolute(): raise ValueError("主密钥路径必须是绝对路径") bootstrap = DatabaseBootstrap(path.parent) bootstrap.key_file = path return bootstrap.master_key(create=True) class HostRegistry: def __init__(self, config, store, log_store, *, factory=None, user_isos=None, cloud_templates=None): self.config = config self.store = store self.log_store = log_store self.factory = factory self.user_isos = user_isos or UserISORuntime( store, Path(config["data_dir"])) self._owns_user_isos = user_isos is None self.cloud_templates = cloud_templates or CloudTemplateRuntime( store, Path(config["data_dir"])) self._owns_cloud_templates = cloud_templates is None self.components: dict[str, EngineComponents] = {} self.apps = {} # Constructed by synchronous factories before an event loop may exist. # Bind the lock lazily inside the ASGI lifespan/request loop. self.lock: asyncio.Lock | None = None self.lifespans = AsyncExitStack() async def start(self): if self.lock is None: self.lock = asyncio.Lock() await asyncio.to_thread(self.user_isos.restore_jobs) await asyncio.to_thread(self.cloud_templates.restore_jobs) await self.lifespans.__aenter__() for host_id in await asyncio.to_thread(self.store.list, "engine_host"): await self.get(host_id) async def close(self): await self.lifespans.aclose() if self._owns_user_isos: await asyncio.to_thread(self.user_isos.close) if self._owns_cloud_templates: await asyncio.to_thread(self.cloud_templates.close) async def get(self, host_id: str): if not HOST_ID.fullmatch(host_id): raise EngineAuthError("宿主机标识无效", 400) if self.lock is None: self.lock = asyncio.Lock() async with self.lock: if host_id in self.apps: return self.apps[host_id] from .app import create_app from .config import Settings directory = Path(self.config["data_dir"]) / "hosts" / host_id if self.factory is not None: try: inspect.signature(self.factory).bind( host_id, self.user_isos, self.cloud_templates) except TypeError: try: inspect.signature(self.factory).bind(host_id, self.user_isos) except TypeError: components, app = self.factory(host_id) else: components, app = self.factory(host_id, self.user_isos) else: components, app = self.factory( host_id, self.user_isos, self.cloud_templates) else: components = await asyncio.to_thread(EngineComponents.for_host, self.store, self.log_store, host_id, directory, self.user_isos, self.cloud_templates) settings = Settings(data_dir=directory) app = await asyncio.to_thread(create_app, settings, engine_components=components) self.components[host_id] = components await self.lifespans.enter_async_context(app.router.lifespan_context(app)) self.apps[host_id] = app return app async def json_response(send, status: int, payload): body = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode() await send({"type": "http.response.start", "status": status, "headers": [(b"content-type", b"application/json"), (b"cache-control", b"no-store")]}) await send({"type": "http.response.body", "body": body}) def raw_target(scope) -> str: path = scope.get("raw_path", scope["path"].encode("ascii")).decode("ascii") query = scope.get("query_string", b"") return path + ("?" + query.decode("ascii") if query else "") def request_headers(scope): values = {} for key, value in scope.get("headers", []): name = key.decode("ascii").lower() if name.startswith("x-cloud-") and name in values: raise EngineAuthError("重复的内部认证头") values[name] = value.decode("latin-1") return values class EngineApplication: def __init__(self, config, store, log_store, *, registry=None, introspection=None, updater=None, update_trigger=None): self.config = config self.store = store self.secret = decode_secret(config["service_secret"]) self.registry = registry or HostRegistry(config, store, log_store) self.introspection = introspection project_dir = Path(config.get("project_dir", Path(__file__).resolve().parents[2])) self.updater = updater or Updater( project_dir, store, worker_active=self._update_service_active) self.update_trigger = update_trigger or self._start_update_service # Serializes the transition into update maintenance mode with every # request that can mutate libvirt or durable operation state. The # lock is created lazily so synchronous test/application factories do # not bind it to an event loop that has not started yet. self.update_coordination_lock: asyncio.Lock | None = None self.active_operations: set[str] = set() self.maintenance_tasks: set[asyncio.Task] = set() self.cache_cleanup_event: asyncio.Event | None = None self.cache_cleanup_supervisor: asyncio.Task | None = None self.cache_cleanup_retry_seconds = CACHE_CLEANUP_RETRY_SECONDS # Target-state operations are idempotent individually, but two # different operation IDs can request opposite states at the same # time (for example traffic suspension ending at a period boundary). # Serialize those mutations per VM so the later request always wins # on the hypervisor as well as in the billing database. self.target_locks: dict[tuple[str, str], asyncio.Lock] = {} self.console_check_interval = 15 def _background(self, coroutine) -> asyncio.Task: task = asyncio.create_task(coroutine) self.maintenance_tasks.add(task) task.add_done_callback(self.maintenance_tasks.discard) return task def _wake_cache_cleanup(self) -> bool: if self.cache_cleanup_event is None: return False self.cache_cleanup_event.set() return True def _pending_cache_cleanup_hosts(self) -> dict[str, set[str]]: """Return only hosts named by a durable, committed delete event.""" pending: dict[str, set[str]] = {} for key, raw in self.store.list("user_iso_tombstone").items(): try: tombstone = self._decode_user_iso_tombstone(key, raw) except ValueError: LOG.error("Corrupt user ISO cache cleanup record: key=%s", key) continue if tombstone.get("source_deleted"): for host_id in tombstone["pending_hosts"]: pending.setdefault(host_id, set()).add("user_iso") for key, raw in self.store.list(CACHE_TOMBSTONE_KIND).items(): try: tombstone = self._decode_cloud_template_tombstone(key, raw) except ValueError: LOG.error( "Corrupt cloud template cache cleanup record: key=%s", key) continue if tombstone.get("source_deleted"): for host_id in tombstone["pending_hosts"]: pending.setdefault(host_id, set()).add("cloud_template") return pending async def _launch_cache_cleanup_pass( self, pending: dict[str, set[str]]) -> list[asyncio.Task]: # Coordinate only the transition into remote maintenance. The actual # SSH/libvirt work is tracked in maintenance_tasks, so update apply is # rejected while a deletion pass is mutating any host. async with self._update_lock(): if await asyncio.to_thread(self.updater.active_job): return [] tasks = [] for host_id, kinds in pending.items(): if "user_iso" in kinds: tasks.append(self._background( self._cleanup_host_user_iso_tombstones(host_id))) if "cloud_template" in kinds: tasks.append(self._background( self._cleanup_host_cloud_template_tombstones(host_id))) return tasks async def _wait_cache_cleanup_retry(self) -> None: event = self.cache_cleanup_event if event is None: return try: await asyncio.wait_for( event.wait(), timeout=max(0.01, self.cache_cleanup_retry_seconds)) except asyncio.TimeoutError: pass event.clear() async def _cache_cleanup_loop(self) -> None: """Retry durable delete tombstones without coupling them to reads.""" event = self.cache_cleanup_event if event is None: return while True: await event.wait() event.clear() while True: try: pending = await asyncio.to_thread( self._pending_cache_cleanup_hosts) if not pending: break tasks = await self._launch_cache_cleanup_pass(pending) if tasks: # asyncio.wait never cancels the tracked maintenance # tasks if this supervisor is cancelled at shutdown. await asyncio.wait(tasks) remaining = await asyncio.to_thread( self._pending_cache_cleanup_hosts) if not remaining: break except asyncio.CancelledError: raise except Exception as exc: LOG.warning( "Delete cache cleanup supervisor deferred: type=%s", type(exc).__name__) await self._wait_cache_cleanup_retry() def _target_lock(self, host_id: str, vm_id: str) -> asyncio.Lock: key = (host_id, vm_id) lock = self.target_locks.get(key) if lock is None: lock = asyncio.Lock() self.target_locks[key] = lock return lock async def __call__(self, scope, receive, send): if scope["type"] == "lifespan": await self._lifespan(receive, send) return if scope["type"] == "websocket": try: await self._console(scope, receive, send) except Exception: LOG.warning("Engine console rejected because a dependency failed") await send({"type": "websocket.close", "code": 4403}) return if scope["type"] != "http": return token = None response_started = False async def tracked_send(message): nonlocal response_started if message["type"] == "http.response.start": response_started = True await send(message) try: # Public reverse proxies must never forward these routes. Reject # non-loopback peers even when they have spoofed forwarded headers. client = scope.get("client") if client and client[0] not in {"127.0.0.1", "::1", "testclient"}: raise EngineAuthError("内部引擎仅接受本机服务请求", 403) headers = request_headers(scope) target = raw_target(scope) upload_resource = None if scope["method"] == "POST": if scope["path"] == "/internal/v1/vm-templates/upload": upload_resource = "vm-templates/upload" elif re.fullmatch( r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates", scope["path"]): upload_resource = "rescue-templates" upload = upload_resource is not None if upload: digest = headers.get("x-cloud-content-sha256", "") delegated = await asyncio.to_thread(authenticate, headers, self.secret, scope["method"], target, digest, self.store) self._authorize_route( scope["method"], upload_resource, delegated) checksum = hashlib.sha256() validation = {"complete": False, "valid": False} scope = { **scope, "engine_upload_validation": validation, "engine_upload_digest": digest, } async def verified_receive(): message = await receive() if message["type"] == "http.request": chunk = message.get("body", b"") checksum.update(chunk) if not message.get("more_body", False): validation.update(complete=True, valid=hmac.compare_digest(checksum.hexdigest(), digest)) return message child_receive = verified_receive body = None else: chunks = [] size = 0 while True: message = await receive() if message["type"] == "http.disconnect": return chunk = message.get("body", b"") size += len(chunk) if size > JSON_LIMIT: raise EngineAuthError("内部请求超过 1 MiB", 413) chunks.append(chunk) if not message.get("more_body", False): break body = b"".join(chunks) delegated = await asyncio.to_thread(authenticate, headers, self.secret, scope["method"], target, hashlib.sha256(body).hexdigest(), self.store) delivered = False async def replay_receive(): nonlocal delivered if not delivered: delivered = True return {"type": "http.request", "body": body, "more_body": False} return await receive() child_receive = replay_receive token = TRUSTED_ACTOR.set(delegated) await self._dispatch(scope, child_receive, tracked_send, body, delegated) except asyncio.CancelledError: raise except EngineAuthError as exc: if not response_started: await json_response(send, exc.status, {"error": str(exc)}) except UpdateError as exc: if not response_started: await json_response(send, exc.status, {"error": str(exc)}) except Exception as exc: LOG.exception("Private engine request failed") if not response_started: await json_response(send, 503, {"error": "引擎服务暂时不可用,请检查引擎日志"}) finally: if token is not None: TRUSTED_ACTOR.reset(token) @staticmethod def _authorize_route(method: str, resource: str, delegated: dict): paths = READ_PATHS if method == "GET" else WRITE_PATHS.get(method, ()) if not any(re.fullmatch(pattern, resource) for pattern in paths): raise EngineAuthError("内部资源不存在", 404) administration = (resource.startswith("host") or resource.startswith("vm-templates/") or resource.startswith("rescue-templates/")) administration_write = method != "GET" and (administration or resource == "rescue-templates") if administration and not delegated["is_admin"]: raise EngineAuthError("仅管理员可访问此资源", 403) if method != "GET" and not delegated["can_manage"]: raise EngineAuthError("当前账户不允许操作", 403) if administration_write and not delegated["is_super_admin"]: raise EngineAuthError("仅超级管理员可修改宿主机和模版", 403) async def _dispatch(self, scope, receive, send, body, delegated): path = scope["path"] if path.startswith("/internal/v1/update/"): await self._update_dispatch(scope["method"], path, body, delegated, send) return method = scope["method"] operation_reconcile = ( method == "GET" and re.fullmatch(r"/internal/v1/operations/[A-Za-z0-9_-]{8,80}", path) is not None and scope.get("query_string", b"") != b"reconcile=false" ) firewall_reconcile = False if operation_reconcile: operation_id = path.rsplit("/", 1)[-1] raw_operation = await asyncio.to_thread( self.store.get, "operation", operation_id) try: stored_operation = json.loads(raw_operation) except (TypeError, ValueError): stored_operation = None firewall_reconcile = ( isinstance(stored_operation, dict) and self._is_firewall_resource( stored_operation.get("resource", "")) ) # A browser upload can spend minutes waiting for request-body chunks. # Its handler acquires the update coordination lock only to reserve and # finally queue the import, so unrelated VM writes remain available. if method == "POST" and path == "/internal/v1/vm-templates/upload": await self._dispatch_regular(scope, receive, send, body, delegated) return firewall_put = ( method == "PUT" and re.fullmatch( rf"/internal/v1/hosts/[A-Za-z0-9][A-Za-z0-9_-]{{0,63}}/" rf"vms/{VM_ID}/firewall", path, ) is not None ) if firewall_put or firewall_reconcile: # Firewall mutations use the per-VM target lock below. They do not # hold the engine-wide update coordination lock while waiting on a # remote libvirt host, so other VMs and read-only VM reads remain # concurrent. A temporary marker closes the race with an update # starting between the maintenance check and durable operation # reservation, without serializing the remote host call. If an # update already owns the coordination lock, reject before placing # that marker; there is no await between this check and the add. if (self.update_coordination_lock is not None and self.update_coordination_lock.locked()): raise EngineAuthError( "面板正在更新,暂时不能核对或修改防火墙", 409) maintenance_gate = "firewall-gate:" + secrets.token_hex(16) self.active_operations.add(maintenance_gate) try: if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError( "面板正在更新,暂时不能核对或修改防火墙", 409) await self._dispatch_regular( scope, receive, send, body, delegated) finally: self.active_operations.discard(maintenance_gate) return if method != "GET" or operation_reconcile: async with self._update_lock(): if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409) await self._dispatch_regular(scope, receive, send, body, delegated) return await self._dispatch_regular(scope, receive, send, body, delegated) def _update_lock(self) -> asyncio.Lock: if self.update_coordination_lock is None: self.update_coordination_lock = asyncio.Lock() return self.update_coordination_lock async def _dispatch_regular(self, scope, receive, send, body, delegated): path = scope["path"] if scope["method"] == "POST" and path == "/internal/v1/hosts/scan": if not delegated["is_super_admin"] or not delegated["can_manage"]: raise EngineAuthError("仅超级管理员可以扫描宿主机", 403) from .app import HostScanBody from .remote_host import RemoteHostManager from pydantic import ValidationError try: value = HostScanBody.model_validate_json(body) except ValidationError as exc: raise EngineAuthError("宿主机扫描参数无效", 422) from exc manager = await asyncio.to_thread(RemoteHostManager, Path(self.config["data_dir"]) / "discovery") try: result = await asyncio.to_thread(manager.scan, value.host, value.port) await json_response(send, 200, result) finally: await asyncio.to_thread(manager.close) return if scope["method"] == "GET" and path == "/internal/v1/health": await asyncio.to_thread(self.store.get, "engine", "readiness") maintenance = await asyncio.to_thread(self.updater.active_job) await json_response(send, 200, { "status": "ok", "accepting_mutations": maintenance is None, }) return cloud_template_match = re.fullmatch( r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|" r"/import-jobs/[A-Za-z0-9_-]+)?)", path) if cloud_template_match is not None: resource = cloud_template_match[1] self._authorize_route(scope["method"], resource, delegated) await self._cloud_template_dispatch( scope["method"], resource, body, send, scope=scope, receive=receive) return user_iso_match = re.fullmatch( r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|" r"/import-jobs/[A-Za-z0-9_-]+)?)", path) if user_iso_match is not None: resource = user_iso_match[1] self._authorize_route(scope["method"], resource, delegated) await self._user_iso_dispatch( scope["method"], resource, body, delegated, send) return op_match = re.fullmatch(r"/internal/v1/operations/([A-Za-z0-9_-]{8,80})", path) if scope["method"] == "GET" and op_match: query = scope.get("query_string", b"") if query not in {b"", b"reconcile=false"}: raise EngineAuthError("操作查询参数无效", 400) operation = await self._operation( op_match[1], delegated, reconcile=query != b"reconcile=false") await json_response(send, 200, self._public_operation(operation)) return match = re.fullmatch(r"/internal/v1/hosts/([A-Za-z0-9][A-Za-z0-9_-]{0,63})/(.+)", path) if match is None: raise EngineAuthError("内部资源不存在", 404) host_id, resource = match.groups() if re.fullmatch( r"user-isos(?:/import|/[0-9a-f]{32}|" r"/import-jobs/[A-Za-z0-9_-]+)?", resource): # User ISO ownership, quota, source bytes and deletion tombstones # are engine-global. Keeping the historical host-prefixed alias # reachable would let DELETE bypass replica GC and make the same # catalog appear to have host-local semantics. raise EngineAuthError("内部资源不存在", 404) self._authorize_route(scope["method"], resource, delegated) app = await self.registry.get(host_id) # Host reads are observations only: they never acquire the engine-wide # mutation/update lock and never repair state as a side effect. Cache # cleanup belongs to the explicit delete workflow that created its # durable tombstone, so unrelated host writes do not run it either. if resource.endswith("/console-ticket"): vm_id = resource.split("/")[1] await self._ticket(host_id, vm_id, delegated, send) return operation = None create = scope["method"] == "POST" and resource == "vms" mutation = scope["method"] == "DELETE" and resource.startswith("vms/") or ( scope["method"] == "PUT" and bool(re.fullmatch( rf"vms/{VM_ID}/firewall", resource))) or ( scope["method"] == "POST" and bool(re.fullmatch( rf"vms/{VM_ID}/(?:actions|reinstall|network|settings|" rf"(?:service|network)-(?:suspend|resume)|" rf"rescue(?:/exit|/force-start)?|" rf"snapshots(?:/[0-9a-f]{{32}}/restore)?)", resource))) if create or mutation: if not OP_ID.fullmatch(delegated.get("operation_id", "")): raise EngineAuthError("虚拟机操作缺少持久幂等编号", 403) if create and not delegated.get("can_provision"): raise EngineAuthError("虚拟机只能通过已确认套餐订单开通", 403) operation, fresh = await self._reserve_operation(host_id, delegated, body, resource=resource, method=scope["method"], target=raw_target(scope)) if operation.get("prevalidation_failed"): await json_response(send, 422, self._public_operation(operation)) return if not fresh: if (operation["status"] in {"running", "pending", "uncertain"} and (self._service_target_method( operation.get("resource", "")) or self._is_network_change_resource( operation.get("resource", "")) or self._is_firewall_resource( operation.get("resource", "")) or self._is_power_action_resource( operation.get("resource", "")))): operation = await self._mutation_operation(operation) if operation["status"] == "succeeded": await json_response(send, 201 if create else 200, {**operation["result"], **self._public_operation(operation)}) else: await json_response(send, 202, self._public_operation(operation)) return if create: delegated["vm_uuid"] = operation["vm_uuid"] self.active_operations.add(operation["operation_id"]) child_scope = {**scope, "path": "/api/v1/" + resource, "raw_path": ("/api/v1/" + resource).encode(), "root_path": ""} if operation is not None and self._is_network_change_resource(resource): def mark_network_mutation_started(): updated = {**operation, "network_mutation_started": True} self.store.set( "operation", operation["operation_id"], json.dumps(updated)) operation.update(network_mutation_started=True) child_scope["virt_cloud_mark_network_mutation_started"] = \ mark_network_mutation_started if operation is None: # Register before saving so all persisted hosts are restored for # unattended sampling after restart, even if the HTTP caller dies. if resource == "host" and scope["method"] == "PUT": await asyncio.to_thread(self.store.set, "engine_host", host_id, json.dumps({"host_id": host_id})) await app(child_scope, receive, send) return messages = [] async def capture(message): messages.append(message) try: target_method = self._service_target_method(operation.get("resource", "")) # Every VM mutation shares the same lock, not just billing target # state. A start/reset/delete that timed out at PHP may still be # executing here when expiry enforcement arrives. Serializing the # later policy call guarantees that its desired state wins. The # reinstall/rescue endpoints register their per-VM pending guard # before returning 202; child endpoints reject every other # mutation until those background jobs remove that guard. async with self._target_lock(host_id, operation["vm_uuid"]): if target_method == "service_suspend": await self._capture_service_was_running_hint( host_id, operation, delegated) elif self._is_network_change_resource( operation.get("resource", "")): await self._capture_network_transition( host_id, operation) elif self._is_vm_delete_operation(operation): await self._capture_delete_cleanup( host_id, operation, delegated) await app(child_scope, receive, capture) status = next(item["status"] for item in messages if item["type"] == "http.response.start") payload = json.loads(b"".join(item.get("body", b"") for item in messages if item["type"] == "http.response.body")) network_preflight_failed = ( self._is_network_change_resource( operation.get("resource", "")) and operation.get("network_mutation_started") is False ) operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else ( "failed" if not create and ( status < 500 or network_preflight_failed) else "uncertain") if operation["status"] == "succeeded": operation["result"] = payload elif operation["status"] == "pending": operation["result"] = payload operation["job_id"] = payload["job_id"] else: operation["error"] = payload.get("detail") or payload.get("error") or "虚拟机开通失败" error_code = payload.get("error_code") if (("/snapshots" in operation.get("resource", "") and error_code in SNAPSHOT_PUBLIC_ERROR_CODES) or (self._is_firewall_resource( operation.get("resource", "")) and error_code in FIREWALL_PUBLIC_ERROR_CODES) or (operation.get("resource", "").endswith("/reinstall") and error_code in REINSTALL_PUBLIC_ERROR_CODES) or (network_preflight_failed and error_code in FIREWALL_PUBLIC_ERROR_CODES)): operation["error_code"] = error_code if self._is_vm_delete_operation(operation): cleanup_code = payload.get("error_code") if cleanup_code in DELETE_PUBLIC_ERROR_CODES: # These codes are emitted only after the child endpoint # has crossed the irreversible domain-delete boundary. # Persist that commit boundary so later polling can # finish exact cleanup without replaying VM DELETE. operation["cleanup_pending"] = "vm_delete" operation["vm_delete_committed"] = True if create: await self._check_absent(operation) operation["finished_at"] = int(time.time()) await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation)) if operation["status"] in {"succeeded", "pending"}: await json_response(send, status, {**payload, **self._public_operation(operation)}) else: await json_response(send, status, {**payload, "operation_id": operation["operation_id"], "status": operation["status"], "safe_to_refund": operation["safe_to_refund"]}) except BaseException as exc: network_preflight_failed = ( self._is_network_change_resource( operation.get("resource", "")) and operation.get("network_mutation_started") is False ) preflight_status = getattr(exc, "status", None) if (isinstance(preflight_status, int) and preflight_status < 500): operation.update( status="failed", error=str(exc), finished_at=int(time.time()), ) elif network_preflight_failed: operation.update( status="failed", error="网络变更尚未开始;前置核对被中断", finished_at=int(time.time()), ) else: operation["status"] = "uncertain" operation["error"] = ( "防火墙结果不确定,请查询原操作核对,不要盲目重放" if self._is_firewall_resource(operation.get("resource", "")) else "开通结果不确定,请查询操作状态,不要重复创建" ) with suppress(Exception): await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation)) raise finally: self.active_operations.discard(operation["operation_id"]) @staticmethod def _cloud_upload_metadata(scope) -> tuple[str, str, str]: query = scope.get("query_string", b"") if (not isinstance(query, bytes) or len(query) > 2048 or re.search(rb"%(?![0-9A-Fa-f]{2})", query) is not None): raise TemplateError("云镜像上传参数无效", 422) try: pairs = parse_qsl( query.decode("ascii"), keep_blank_values=True, strict_parsing=True, max_num_fields=3, errors="strict", ) except (UnicodeDecodeError, ValueError) as exc: raise TemplateError("云镜像上传参数无效", 422) from exc if (len(pairs) != 3 or {key for key, _value in pairs} != {"name", "os_family", "os_version"}): raise TemplateError("云镜像上传参数无效", 422) values = dict(pairs) name = validate_template_name(values["name"]) os_family, os_version = validate_template_metadata( values["os_family"], values["os_version"]) return name, os_family, os_version @staticmethod def _single_upload_header(scope, name: bytes) -> str | None: values = [value.decode("latin-1") for key, value in scope.get("headers", []) if key.lower() == name] if len(values) > 1: raise TemplateError("云镜像上传请求头重复", 400) return values[0] if values else None async def _upload_cloud_template(self, scope, receive, send) -> None: runtime = self.registry.cloud_templates name, os_family, os_version = self._cloud_upload_metadata(scope) content_type = self._single_upload_header(scope, b"content-type") if content_type is None or content_type.lower() != "application/octet-stream": raise TemplateError( "云镜像上传必须使用 application/octet-stream", 415) content_encoding = self._single_upload_header(scope, b"content-encoding") if content_encoding is not None and content_encoding.lower() != "identity": raise TemplateError("云镜像上传内容不能压缩", 415) declared_raw = self._single_upload_header(scope, b"content-length") if declared_raw is not None and re.fullmatch(r"[0-9]+", declared_raw) is None: raise TemplateError("云镜像上传长度无效", 400) declared = int(declared_raw) if declared_raw is not None else None if declared is not None and declared < MIN_DOWNLOAD_BYTES: raise TemplateError("云镜像文件不能小于 1 MiB", 422) if declared is not None and declared > MAX_IMAGE_BYTES: raise TemplateError("云镜像超过 4 GiB 限制", 413) async with self._update_lock(): if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError( "面板正在更新,暂时不能执行虚拟化写操作", 409) reservation = await asyncio.to_thread( runtime.reserve_upload, name, os_family, os_version) reservation_id = reservation["reservation_id"] maximum = reservation["max_bytes"] if declared is not None and declared > maximum: await asyncio.to_thread(runtime.cancel_upload, reservation_id) raise TemplateError("面板云镜像持久目录空间不足", 507) image_path: Path | None = None handed_off = False writer: _CloudUploadWriter | None = None try: writer = await asyncio.to_thread( _CloudUploadWriter, reservation["directory"]) image_path = writer.path checksum = hashlib.sha256() size = 0 deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS try: while True: remaining = deadline - time.monotonic() if remaining <= 0: raise TemplateError("云镜像上传超时", 408) try: message = await asyncio.wait_for( receive(), timeout=min(CLOUD_UPLOAD_IDLE_SECONDS, remaining), ) except TimeoutError as exc: raise TemplateError("云镜像上传超时", 408) from exc if message["type"] == "http.disconnect": raise TemplateError("云镜像上传已中断", 400) if message["type"] != "http.request": raise TemplateError("云镜像上传请求无效", 400) chunk = message.get("body", b"") size += len(chunk) if declared is not None and size > declared: raise TemplateError("云镜像上传内容超过声明长度", 400) if size > maximum: if maximum < MAX_IMAGE_BYTES: raise TemplateError( "面板云镜像持久目录空间不足", 507) raise TemplateError("云镜像超过 4 GiB 限制", 413) view = memoryview(chunk) for offset in range(0, len(view), CLOUD_UPLOAD_WRITE_BYTES): if time.monotonic() >= deadline: raise TemplateError("云镜像上传超时", 408) part = view[offset:offset + CLOUD_UPLOAD_WRITE_BYTES] await asyncio.to_thread(writer.write, part) checksum.update(part) if not message.get("more_body", False): break await asyncio.to_thread(writer.sync_and_close) except OSError as exc: if exc.errno in {errno.ENOSPC, errno.EDQUOT}: raise TemplateError( "面板云镜像持久目录空间不足", 507) from exc raise if declared is not None and size != declared: raise TemplateError("云镜像上传内容不完整", 400) if size < MIN_DOWNLOAD_BYTES: raise TemplateError("云镜像文件不能小于 1 MiB", 422) validation = scope.get("engine_upload_validation", {}) expected_digest = scope.get("engine_upload_digest", "") digest = checksum.hexdigest() if (not validation.get("complete") or not validation.get("valid") or not hmac.compare_digest(digest, expected_digest)): raise EngineAuthError("云镜像摘要与签名不匹配", 401) virtual_size = await asyncio.to_thread( verify_qcow2_image, image_path) # Recheck the update state at the commit boundary. An upload # admitted before a service transition must not enqueue work # against code that is being replaced. async with self._update_lock(): if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError( "面板正在更新,暂时不能执行虚拟化写操作", 409) result = await asyncio.to_thread( runtime.queue_upload, reservation_id, image_path, digest, virtual_size, ) handed_off = True await json_response(send, 202, result) finally: if not handed_off: if writer is not None: with suppress(OSError): await asyncio.to_thread(writer.close) if image_path is not None: with suppress(OSError): await asyncio.to_thread(image_path.unlink, missing_ok=True) await asyncio.to_thread(runtime.cancel_upload, reservation_id) async def _cloud_template_dispatch( self, method: str, resource: str, body: bytes | None, send, *, scope=None, receive=None) -> None: runtime = self.registry.cloud_templates try: if method == "GET" and resource == "vm-templates": records = await asyncio.to_thread(runtime.list) await json_response(send, 200, {"templates": records}) return if method == "POST" and resource == "vm-templates/import": from .app import ImportTemplateBody from pydantic import ValidationError try: value = ImportTemplateBody.model_validate_json(body or b"") except ValidationError as exc: raise TemplateError("云镜像导入参数无效", 422) from exc result = await asyncio.to_thread( runtime.queue_import, value.url, value.name, value.sha256) await json_response(send, 202, result) return if method == "POST" and resource == "vm-templates/upload": if scope is None or receive is None: raise EngineAuthError("云镜像上传请求无效", 400) await self._upload_cloud_template(scope, receive, send) return job_match = re.fullmatch( r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource) if method == "GET" and job_match is not None: result = await asyncio.to_thread( runtime.get_job, job_match[1]) await json_response(send, 200, result) return template_match = re.fullmatch( r"vm-templates/([A-Za-z0-9_-]+)", resource) if method == "PATCH" and template_match is not None: from .app import RenameTemplateBody from pydantic import ValidationError try: value = RenameTemplateBody.model_validate_json(body or b"") except ValidationError as exc: raise TemplateError("云镜像名称无效", 422) from exc record = await asyncio.to_thread( runtime.rename, template_match[1], value.name) await json_response(send, 200, {"template": record}) return if method == "DELETE" and template_match is not None: from .app import DeleteTemplateBody from pydantic import ValidationError try: value = DeleteTemplateBody.model_validate_json(body or b"") except ValidationError as exc: raise TemplateError("云镜像删除参数无效", 422) from exc template_id = template_match[1] tombstone_key, tombstone = await asyncio.to_thread( self._delete_cloud_template_source, template_id, value.created_at) if not self._wake_cache_cleanup(): for host_id in tombstone["pending_hosts"]: self._background( self._cleanup_cloud_template_tombstone_on_host( tombstone_key, tombstone, host_id)) await json_response(send, 200, { "deleted": True, "cache_cleanup_pending": bool(tombstone["pending_hosts"]), }) return raise EngineAuthError("内部资源不存在", 404) except TemplateError as exc: await json_response(send, exc.status, {"error": str(exc)}) async def _user_iso_dispatch(self, method: str, resource: str, body: bytes | None, delegated: dict, send) -> None: runtime = self.registry.user_isos user_id = delegated["user_id"] try: if method == "GET" and resource == "user-isos": result = await asyncio.to_thread(runtime.list_for, user_id) await json_response(send, 200, result) return if method == "POST" and resource == "user-isos/import": from .app import ImportUserISOBody from pydantic import ValidationError try: value = ImportUserISOBody.model_validate_json(body or b"") except ValidationError as exc: raise TemplateError("ISO 导入参数无效", 422) from exc result = await asyncio.to_thread( runtime.queue_import, user_id, value.url, value.name, value.sha256) await json_response(send, 202, result) return job_match = re.fullmatch( r"user-isos/import-jobs/([A-Za-z0-9_-]+)", resource) if method == "GET" and job_match is not None: result = await asyncio.to_thread( runtime.get_job_for, user_id, job_match[1]) await json_response(send, 200, result) return iso_match = re.fullmatch(r"user-isos/([0-9a-f]{32})", resource) if iso_match is not None and method == "GET": record = await asyncio.to_thread( runtime.get_for, user_id, iso_match[1]) await json_response( send, 200, {"iso": runtime.public_record(record)}) return if iso_match is not None and method == "DELETE": iso_id = iso_match[1] tombstone_key, tombstone = await asyncio.to_thread( self._delete_user_iso_source, user_id, iso_id) if not self._wake_cache_cleanup(): for host_id in tombstone["pending_hosts"]: self._background( self._cleanup_user_iso_tombstone_on_host( tombstone_key, tombstone, host_id)) await json_response(send, 200, { "deleted": True, "cache_cleanup_pending": bool(tombstone["pending_hosts"]), }) return raise EngineAuthError("内部资源不存在", 404) except TemplateError as exc: await json_response(send, exc.status, {"error": str(exc)}) def _delete_cloud_template_source( self, template_id: str, expected_created_at: int) -> tuple[str, dict]: """Commit global source deletion and durable per-host cache GC.""" runtime = self.registry.cloud_templates with runtime.lock: raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id) if raw is not None: tombstone = self._decode_cloud_template_tombstone( template_id, raw) if tombstone["template_created_at"] != expected_created_at: raise TemplateError( "云镜像版本已变化,请刷新后重试", 409) if tombstone["source_deleted"]: return template_id, tombstone try: record = runtime.get(template_id) except TemplateError as exc: if raw is None or exc.status != 404: raise tombstone["source_deleted"] = True self.store.set( CACHE_TOMBSTONE_KIND, template_id, json.dumps(tombstone, separators=(",", ":")), ) return template_id, tombstone source_volume_name = record.get("volume_name") if source_volume_name != template_volume_name(template_id): raise TemplateError("云镜像目录记录损坏,无法安全删除", 500) if record.get("created_at") != expected_created_at: raise TemplateError("云镜像版本已变化,请刷新后重试", 409) host_ids = sorted( set(self.store.list("engine_host")) | set(self.registry.components) ) tombstone = { "deletion_id": uuid.uuid4().hex, "template_id": template_id, "template_created_at": expected_created_at, "source_volume_name": source_volume_name, "pending_hosts": host_ids, "source_deleted": False, "created_at": int(time.time()), } encoded = json.dumps(tombstone, separators=(",", ":")) self.store.set(CACHE_TOMBSTONE_KIND, template_id, encoded) try: # Re-enters runtime.lock, which is intentionally an RLock. runtime.delete(template_id) except Exception: # A database DELETE acknowledgement can be lost before or # after commit. The catalog alone is not sufficient proof of # rollback because source unlink happens first. Reconcile both # durable halves: only an intact catalog+source pair cancels # this marker; a missing source completes retirement and keeps # host GC durable. if self.store.get( CACHE_TOMBSTONE_KIND, template_id) == encoded: reconciled = runtime._reconcile_cache_tombstone( template_id, tombstone.copy()) if (reconciled is not None and reconciled["source_deleted"]): return template_id, reconciled raise tombstone["source_deleted"] = True self.store.set( CACHE_TOMBSTONE_KIND, template_id, json.dumps(tombstone, separators=(",", ":")), ) return template_id, tombstone @staticmethod def _decode_cloud_template_tombstone(key: str, raw: str) -> dict: try: value = json.loads(raw) deletion_id = value["deletion_id"] template_id = value["template_id"] template_created_at = value["template_created_at"] pending = value["pending_hosts"] except (KeyError, TypeError, ValueError) as exc: raise ValueError("cloud template tombstone is corrupt") from exc try: expected_volume_name = template_volume_name(template_id) except TemplateError as exc: raise ValueError("cloud template tombstone is corrupt") from exc if (not isinstance(deletion_id, str) or not re.fullmatch(r"[0-9a-f]{32}", deletion_id) or key != template_id or isinstance(template_created_at, bool) or not isinstance(template_created_at, int) or template_created_at < 1 or value.get("source_volume_name") != expected_volume_name or not isinstance(pending, list) or any(not isinstance(host_id, str) or not HOST_ID.fullmatch(host_id) for host_id in pending) or len(set(pending)) != len(pending) or not isinstance(value.get("source_deleted"), bool)): raise ValueError("cloud template tombstone is corrupt") return value async def _cleanup_cloud_template_tombstone_on_host( self, key: str, tombstone: dict, host_id: str) -> None: if (not tombstone.get("source_deleted") or host_id not in tombstone["pending_hosts"]): return try: await self.registry.get(host_id) components = self.registry.components[host_id] profile = await asyncio.to_thread(components.manager.current) if profile is None or profile.get("key_locked"): return service = self._service(profile) await asyncio.to_thread( self._cleanup_cloud_template_tombstone_on_host_sync, key, tombstone["deletion_id"], host_id, service, components) except Exception as exc: LOG.info( "Cloud template cache cleanup deferred: host=%s type=%s", host_id, type(exc).__name__) def _cleanup_cloud_template_tombstone_on_host_sync( self, key: str, deletion_id: str, host_id: str, service, components: EngineComponents) -> None: """Revalidate the deletion generation at the libvirt commit point. Duplicate cleanup tasks can outlive the tombstone snapshot that scheduled them. Holding the shared runtime lock across the final re-read, libvirt deletion and marker update prevents an obsolete task from deleting a replica created by a same-content reimport. """ runtime = self.registry.cloud_templates with runtime.lock: raw = self.store.get(CACHE_TOMBSTONE_KIND, key) if raw is None: return value = self._decode_cloud_template_tombstone(key, raw) if (value["deletion_id"] != deletion_id or not value["source_deleted"] or host_id not in value["pending_hosts"]): return try: service.delete_cloud_template_replicas( value["source_volume_name"]) except Exception as exc: from .libvirt_service import VMError # Only an accessible pool with no matching volumes is an # idempotent success. A missing/misconfigured storage pool is # also reported as 404 by libvirt_service and must keep this # host pending so an old replica cannot escape reclamation. if (not isinstance(exc, VMError) or exc.status != 404 or exc.error_code != "cloud_template_cache_not_found"): raise components.store.delete( "cloud_template_replica", value["template_id"]) value["pending_hosts"] = [ candidate for candidate in value["pending_hosts"] if candidate != host_id ] # Retain the terminal record so a response lost after source # deletion remains retryable until an intentional reimport. self.store.set( CACHE_TOMBSTONE_KIND, key, json.dumps(value, separators=(",", ":")), ) async def _cleanup_host_cloud_template_tombstones( self, host_id: str) -> None: rows = await asyncio.to_thread( self.store.list, CACHE_TOMBSTONE_KIND) for key, raw in rows.items(): try: tombstone = self._decode_cloud_template_tombstone(key, raw) except ValueError: LOG.error( "Corrupt cloud template cache cleanup record: key=%s", key) continue if host_id in tombstone["pending_hosts"]: await self._cleanup_cloud_template_tombstone_on_host( key, tombstone, host_id) def _delete_user_iso_source( self, user_id: int, iso_id: str) -> tuple[str, dict]: """Commit one source deletion and its remote-GC tombstone atomically. The shared runtime lock serializes the whole transition. In particular, a second DELETE can only observe the first request's committed tombstone; it must never remove that durable cleanup record after the catalog has disappeared. """ runtime = self.registry.user_isos key = f"{user_id}:{iso_id}" with runtime.lock: raw = self.store.get("user_iso_tombstone", key) if raw is not None: tombstone = self._decode_user_iso_tombstone(key, raw) if tombstone["source_deleted"]: return key, tombstone try: record = runtime.get_for(user_id, iso_id) except TemplateError as exc: if raw is None or exc.status != 404: raise # The source/catalog removal committed but persisting the # final flag failed. Preserve and advance the existing GC # record instead of erasing it on a repeated DELETE. tombstone["source_deleted"] = True self.store.set("user_iso_tombstone", key, json.dumps( tombstone, separators=(",", ":"))) if not tombstone["pending_hosts"]: self.store.delete("user_iso_tombstone", key) return key, tombstone host_ids = self._user_iso_replica_hosts(user_id, record) tombstone = { "user_id": user_id, "iso_id": iso_id, "volume_name": record["volume_name"], "pending_hosts": host_ids, "source_deleted": False, "created_at": int(time.time()), } encoded = json.dumps(tombstone, separators=(",", ":")) self.store.set("user_iso_tombstone", key, encoded) try: # Re-enters runtime.lock; it is intentionally an RLock. runtime.delete_for(user_id, iso_id) except Exception: # Only erase the exact provisional record written above. A # later committed tombstone must never be removed by a failed # or repeated request. if self.store.get("user_iso_tombstone", key) == encoded: self.store.delete("user_iso_tombstone", key) raise tombstone["source_deleted"] = True self.store.set("user_iso_tombstone", key, json.dumps( tombstone, separators=(",", ":"))) if not host_ids: self.store.delete("user_iso_tombstone", key) return key, tombstone def _user_iso_replica_hosts(self, user_id: int, record: dict) -> list[str]: replica_key = f"{user_id}:{record['id']}" result = [] host_ids = set(self.store.list("engine_host")) | set(self.registry.components) for host_id in host_ids: raw = HostStore(self.store, host_id).get( "user_iso_replica", replica_key) if raw is None: continue # The exact owner/ISO marker key is already proof that this host # may contain the deterministic volume. Reuse requires a strict # ready marker, but deletion must fail safe for old or partially # damaged marker schemas as well. result.append(host_id) return sorted(result) @staticmethod def _decode_user_iso_tombstone(key: str, raw: str) -> dict: from .user_isos import user_iso_volume_name try: value = json.loads(raw) user_id = value["user_id"] iso_id = value["iso_id"] pending = value["pending_hosts"] except (KeyError, TypeError, ValueError) as exc: raise ValueError("user ISO tombstone is corrupt") from exc if (isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1 or key != f"{user_id}:{iso_id}" or value.get("volume_name") != user_iso_volume_name(iso_id) or not isinstance(pending, list) or any(not isinstance(host_id, str) or not HOST_ID.fullmatch(host_id) for host_id in pending) or len(set(pending)) != len(pending) or not isinstance(value.get("source_deleted"), bool)): raise ValueError("user ISO tombstone is corrupt") return value async def _cleanup_user_iso_tombstone_on_host( self, key: str, tombstone: dict, host_id: str) -> None: if not tombstone.get("source_deleted") or host_id not in tombstone["pending_hosts"]: return try: await self.registry.get(host_id) components = self.registry.components[host_id] profile = await asyncio.to_thread(components.manager.current) if profile is None or profile.get("key_locked"): return service = self._service(profile) try: await asyncio.to_thread( service.delete_user_iso, tombstone["user_id"], tombstone["volume_name"]) except Exception as exc: from .libvirt_service import VMError # A root pool/configuration 404 is not proof that the exact # per-user volume is absent. Keep the tombstone until the # service returns the narrow idempotency code. if (not isinstance(exc, VMError) or exc.status != 404 or exc.error_code != "user_iso_volume_not_found"): LOG.info( "User ISO cache cleanup remains pending: host=%s type=%s", host_id, type(exc).__name__) return await asyncio.to_thread( components.store.delete, "user_iso_replica", f"{tombstone['user_id']}:{tombstone['iso_id']}") await asyncio.to_thread( self._complete_user_iso_tombstone_host, key, host_id) except Exception as exc: LOG.info("User ISO cache cleanup deferred: host=%s type=%s", host_id, type(exc).__name__) def _complete_user_iso_tombstone_host(self, key: str, host_id: str) -> None: runtime = self.registry.user_isos with runtime.lock: raw = self.store.get("user_iso_tombstone", key) if raw is None: return value = self._decode_user_iso_tombstone(key, raw) value["pending_hosts"] = [ candidate for candidate in value["pending_hosts"] if candidate != host_id ] if value["pending_hosts"]: self.store.set("user_iso_tombstone", key, json.dumps( value, separators=(",", ":"))) else: self.store.delete("user_iso_tombstone", key) async def _cleanup_host_user_iso_tombstones(self, host_id: str) -> None: rows = await asyncio.to_thread(self.store.list, "user_iso_tombstone") for key, raw in rows.items(): try: tombstone = self._decode_user_iso_tombstone(key, raw) except ValueError: LOG.error("Corrupt user ISO cache cleanup record: key=%s", key) continue if host_id in tombstone["pending_hosts"]: await self._cleanup_user_iso_tombstone_on_host( key, tombstone, host_id) @staticmethod def _update_object(body: bytes | None, *, empty: bool = False) -> dict: if (body is None or body == b"") and empty: return {} try: value = json.loads(body or b"") except (TypeError, ValueError, UnicodeError) as exc: raise UpdateError("在线更新参数必须是 JSON 对象", 422) from exc if not isinstance(value, dict): raise UpdateError("在线更新参数必须是 JSON 对象", 422) return value @staticmethod def _public_update_job(job: dict) -> dict: allowed = { "job_id", "status", "phase", "created_at", "started_at", "finished_at", "updated", "old_commit", "new_commit", "rollback_ref", "error", } return {key: value for key, value in job.items() if key in allowed} def _long_running_host_job(self) -> bool: cloud_templates = getattr(self.registry, "cloud_templates", None) if (self.maintenance_tasks or self.registry.user_isos.has_active_jobs() or (cloud_templates is not None and cloud_templates.has_active_jobs())): return True for components in self.registry.components.values(): kinds = [ "rescue_import_job", "reinstall_job", "rescue_job", "snapshot_job", ] # A child app without the shared runtime can still run the legacy # host-scoped importer. Production children use the global runtime; # stale pre-upgrade import_job rows must not block updates forever. if components.cloud_templates is None: kinds.append("import_job") for kind in kinds: for raw in components.store.list(kind).values(): try: status = json.loads(raw).get("status") except (AttributeError, TypeError, ValueError): return True if status in {"queued", "running", "pending"}: return True return False async def _update_dispatch(self, method: str, path: str, body: bytes | None, delegated: dict, send) -> None: if not (delegated.get("is_admin") and delegated.get("is_super_admin") and delegated.get("can_manage")): raise EngineAuthError("仅超级管理员可以管理在线更新", 403) if path == "/internal/v1/update/settings" and method == "GET": await json_response(send, 200, await asyncio.to_thread(self.updater.settings)) return if path == "/internal/v1/update/settings" and method == "PUT": value = self._update_object(body) if set(value) - {"access_token", "clear_token"}: raise UpdateError("在线更新设置包含未知字段", 422) clear = value.get("clear_token", False) token = value.get("access_token") if not isinstance(clear, bool) or (token is not None and not isinstance(token, str)): raise UpdateError("在线更新设置格式无效", 422) if clear and token not in {None, ""}: raise UpdateError("不能同时保存和删除访问令牌", 422) if clear: await asyncio.to_thread(self.updater.clear_token) elif token not in {None, ""}: await asyncio.to_thread(self.updater.set_token, token) elif "access_token" not in value: raise UpdateError("没有需要保存的在线更新设置", 422) await json_response(send, 200, await asyncio.to_thread(self.updater.settings)) return if path == "/internal/v1/update/check" and method == "POST": value = self._update_object(body, empty=True) if value: raise UpdateError("检查更新不接受额外参数", 422) if await asyncio.to_thread(self.updater.active_job): raise UpdateError("已有更新任务正在执行", 409) await json_response(send, 200, await asyncio.to_thread(self.updater.check)) return if path == "/internal/v1/update/apply" and method == "POST": value = self._update_object(body) if set(value) != {"expected_current_commit", "expected_remote_commit"}: raise UpdateError("请先检查更新并提交已确认的版本", 422) if not all(isinstance(value[key], str) for key in value): raise UpdateError("更新版本标识无效", 422) async with self._update_lock(): if self.active_operations or await asyncio.to_thread(self._long_running_host_job): raise UpdateError("仍有虚拟机任务正在执行,请等待任务完成后再更新", 409) job = await asyncio.to_thread( self.updater.create_job, expected_current_commit=value["expected_current_commit"], expected_remote_commit=value["expected_remote_commit"], ) try: await asyncio.to_thread(self.update_trigger) job = await self._wait_update_worker_started(job["job_id"]) except UpdateServiceUncertain: raise except UpdateError as exc: await asyncio.to_thread( self.updater.fail_job, job["job_id"], "独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", ) raise exc except Exception as exc: await asyncio.to_thread( self.updater.fail_job, job["job_id"], "独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", ) raise UpdateError( "独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503 ) from exc await json_response(send, 202, self._public_update_job(job)) return job_match = re.fullmatch( r"/internal/v1/update/jobs/([A-Za-z0-9_-]{16,64})", path) if method == "GET" and job_match: job = await asyncio.to_thread(self.updater.get_job, job_match[1]) if job is None: raise UpdateError("更新任务不存在", 404) await json_response(send, 200, self._public_update_job(job)) return raise EngineAuthError("内部资源不存在", 404) async def _wait_update_worker_started(self, job_id: str) -> dict: """Confirm the one-shot claimed the durable job, not merely its queue.""" deadline = time.monotonic() + 15 while time.monotonic() < deadline: job = await asyncio.to_thread(self.updater.get_job, job_id) if job is None: raise UpdateError("独立更新服务未生成任务状态", 503) if job.get("status") in {"running", "succeeded"}: return job if job.get("status") == "failed": raise UpdateError("独立更新服务启动后失败,请查看更新服务日志", 503) if job.get("status") == "blocked": raise UpdateServiceUncertain( "更新任务中断且维护锁仍在生效,请通过 SSH 检查更新服务和回滚引用", 503, ) await asyncio.sleep(0.1) # Stop any very-late process before releasing the maintenance gate and # marking the job failed. This prevents an unclaimed worker from # changing code after ordinary VM mutations have resumed. await asyncio.to_thread(self._stop_update_service) raise UpdateError("独立更新服务未在限定时间内启动,请查看 systemd 日志", 503) @staticmethod def _start_update_service() -> None: try: result = subprocess.run( ["/usr/bin/systemctl", "--no-block", "start", "virt-cloud-update.service"], cwd="/", env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, capture_output=True, text=True, timeout=15, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise UpdateError( "独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503 ) from exc if result.returncode != 0: raise UpdateError( "独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503) @staticmethod def _update_service_active() -> bool | None: try: result = subprocess.run( ["/usr/bin/systemctl", "show", "--property=ActiveState", "--value", "virt-cloud-update.service"], cwd="/", env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, capture_output=True, text=True, timeout=5, check=False, ) except (OSError, subprocess.TimeoutExpired): return None if result.returncode != 0: return None state = result.stdout.strip() if state in {"active", "activating", "deactivating", "reloading"}: return True if state in {"inactive", "failed"}: return False return None @staticmethod def _stop_update_service() -> None: try: result = subprocess.run( ["/usr/bin/systemctl", "stop", "virt-cloud-update.service"], cwd="/", env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, capture_output=True, text=True, timeout=30, check=False, ) except (OSError, subprocess.TimeoutExpired) as exc: raise UpdateServiceUncertain( "无法停止未确认的更新服务,请立即检查 systemd 状态", 503) from exc if result.returncode != 0: raise UpdateServiceUncertain( "无法停止未确认的更新服务,请立即检查 systemd 状态", 503) async def _reserve_operation(self, host_id, delegated, body, *, resource="vms", method="POST", target=""): op_id = delegated["operation_id"] # Even rejected signed provisioning requests have a durable outcome. # A worker that loses this HTTP response can release its frozen balance # only by querying this same operation, without replaying the request. from .app import ( ActionBody, CreateCloudBody, CreateISOBody, FirewallPolicyBody, ) from pydantic import ValidationError create = resource == "vms" value = {} error = None try: value = json.loads(body) if body else {} if not isinstance(value, dict): raise ValueError("body must be an object") if create: (CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value) elif self._is_power_action_resource(resource): # Persist only the reviewed action name. The original body # may contain future/private fields and must never become a # reconciliation instruction after a process restart. value = ActionBody.model_validate(value).model_dump() elif self._is_firewall_resource(resource): value = FirewallPolicyBody.model_validate(value).model_dump() from .libvirt_service import ( FIREWALL_FILTER_COMPILER_VERSION, firewall_policy_digest, ) expected_digest = firewall_policy_digest( value["enabled"], value["rules"]) if not hmac.compare_digest( value["policy_digest"], expected_digest): raise ValueError("firewall digest mismatch") if value.get("guest_ssh_key_id") is not None or value.get("ssh_key_id") is not None: error = "公钥必须由 PHP 校验用户归属后传递,不能使用旧公钥 ID" except (ValueError, AttributeError, TypeError, ValidationError): value = {} error = "虚拟机参数校验失败" record = {"operation_id": op_id, "host_id": host_id, "user_id": delegated["user_id"], "request_hash": hmac.new(self.secret, method.encode() + b"\n" + target.encode() + b"\n" + (body or b""), hashlib.sha256).hexdigest(), "vm_uuid": str(uuid.uuid4()) if create else str(uuid.UUID(resource.split("/")[1])), "vm_name": value.get("name"), "kind": "create" if create else "mutation", "actor_can_manage": bool(delegated.get("can_manage")), "actor_is_admin": bool(delegated.get("is_admin")), "resource": resource, "method": method, "status": "running", "result": None, "safe_to_refund": False, "created_at": int(time.time())} if create: # New PHP-backed creates always receive the reserved UUID below. # Keep this durable marker so cleanup reconciliation checks the # UUID-derived resource candidates, while old in-flight records # continue to use their historical display-name disk convention. record["resource_naming"] = "uuid-v1" if self._is_power_action_resource(resource) and not error: record["target_action"] = value["action"] if resource.endswith("/service-resume"): # Store only the allowlisted target-state option. It is needed to # replay the exact same convergence request after a lost response; # never persist arbitrary request bodies in the operation table. record["target_options"] = { "start_if_suspended": value.get("start_if_suspended") is True, } if self._is_network_change_resource(resource): # The child flips this durable checkpoint immediately before # claiming the target IP. Until then, a crash or 5xx cannot make # the IP transition ambiguous. record["network_mutation_started"] = False target_cidr = value.get("public_ip_cidr") if isinstance(target_cidr, str): # This is the only request field retained for reconciliation. # Credentials or arbitrary mutation bodies are never persisted. record["network_target_cidr"] = target_cidr target_pool_key = value.get("public_pool_key") if (isinstance(target_pool_key, str) and re.fullmatch(r"[0-9a-f]{32}", target_pool_key)): record["network_target_pool_key"] = target_pool_key if method == "DELETE" and resource.startswith("vms/"): query = dict(parse_qsl(urlsplit(target).query, keep_blank_values=True)) record["delete_disks"] = query.get("delete_disks", "").casefold() in { "1", "true", "yes", "on", } if self._is_firewall_resource(resource) and not error: record["firewall_target"] = { "enabled": value["enabled"], "revision": value["revision"], "policy_digest": value["policy_digest"], "rules": value["rules"], "filter_version": FIREWALL_FILTER_COMPILER_VERSION, } if error: # No service, task, VM or volume mutation has been invoked. record.update(status="failed", prevalidation_failed=True, error=error, safe_to_refund=create, finished_at=int(time.time())) if self._is_firewall_resource(resource): record["error_code"] = "firewall_rule_invalid" fresh = await asyncio.to_thread(self.store.insert, "operation", op_id, json.dumps(record)) if fresh: return record, True previous = await self._operation(op_id, delegated, reconcile=False) if previous["host_id"] != host_id or previous["user_id"] != delegated["user_id"] or previous["request_hash"] != record["request_hash"]: raise EngineAuthError("相同开通操作编号的请求内容不一致", 409) return previous, False async def _operation(self, op_id, delegated, *, reconcile=True): raw = await asyncio.to_thread(self.store.get, "operation", op_id) if raw is None: raise EngineAuthError("开通操作不存在", 404) record = json.loads(raw) if record["user_id"] != delegated["user_id"] and not delegated["is_admin"]: raise EngineAuthError("开通操作不存在", 404) # Never automatically create again. Recover a known UUID if libvirt # committed while the response or process was lost. if reconcile and record.get("kind") == "mutation" and record["status"] in {"running", "pending", "uncertain"}: return await self._mutation_operation(record) if (reconcile and record["status"] in {"running", "uncertain"} and op_id not in self.active_operations and (record["status"] == "uncertain" or int(time.time()) - record.get("created_at", 0) >= 120)): await self.registry.get(record["host_id"]) components = self.registry.components[record["host_id"]] profile = components.manager.current() if profile and not profile.get("key_locked"): try: vm = await asyncio.to_thread(self._service(profile).get_vm, record["vm_uuid"]) if vm.get("state") != "running" or not vm.get("managed"): return record from .metrics import host_scope await asyncio.to_thread(components.state.users.assign_vm, host_scope(profile), record["vm_uuid"], record["user_id"]) record.update(status="succeeded", result=vm, safe_to_refund=False) await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record)) except Exception: # A missing VM can be a transient connection failure or an # operation still in progress. It is not permission to retry. await self._check_absent(record) await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record)) return record async def _mutation_operation(self, record): app = await self.registry.get(record["host_id"]) components = self.registry.components[record["host_id"]] if (self._is_vm_delete_operation(record) and record.get("cleanup_pending") in { "public_ip_claim", "vm_delete", }): return await self._delete_cleanup_operation(record, components) if self._is_firewall_resource(record.get("resource", "")): return await self._firewall_operation(record, components) if self._is_network_change_resource(record.get("resource", "")): return await self._network_change_operation(record, components) if self._is_power_action_resource(record.get("resource", "")): return await self._power_action_operation(record, components) target_method = self._service_target_method(record.get("resource", "")) if target_method is not None: if record["operation_id"] in self.active_operations: return record if (target_method == "service_suspend" and not isinstance(record.get("service_was_running_hint"), bool)): # Suspending without a durable pre-stop state can permanently # lose whether a later paid renewal is allowed to restart the # guest. Fail closed instead of replaying a destructive target # after the database recovers. record.update( status="failed", error="停机前运行状态未安全保存,已取消服务暂停", finished_at=int(time.time()), ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record profile = components.manager.current() if profile is None or profile.get("key_locked"): return record if (not record.get("actor_can_manage") or (not record.get("actor_is_admin") and not components.state.users.owns_vm( self._host_scope(profile), record["vm_uuid"], record["user_id"]))): record.update(status="failed", error="虚拟机不存在", finished_at=int(time.time())) await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record)) return record self.active_operations.add(record["operation_id"]) try: async with self._target_lock(record["host_id"], record["vm_uuid"]): options = record.get("target_options", {}) \ if target_method == "service_resume" else {} if not isinstance(options, dict) or any( key != "start_if_suspended" for key in options): options = {} result = await asyncio.to_thread( getattr(self._service(profile), target_method), record["vm_uuid"], **options) if (target_method == "service_suspend" and isinstance(record.get("service_was_running_hint"), bool)): result["service_was_running"] = ( bool(result.get("service_was_running")) or record["service_was_running_hint"]) except Exception as exc: from .libvirt_service import VMError if isinstance(exc, VMError) and exc.status < 500: record.update(status="failed", error=str(exc), finished_at=int(time.time())) else: record.update(status="uncertain", error="服务状态尚未确认,引擎将安全重试") else: record.update(status="succeeded", result=result, error=None, finished_at=int(time.time())) finally: self.active_operations.discard(record["operation_id"]) await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record)) return record if self._is_vm_delete_operation(record): return await self._uncertain_delete_operation(record) kind = ( "snapshot_job" if "/snapshots" in record["resource"] else "reinstall_job" if record["resource"].endswith("/reinstall") else "rescue_job" if "/rescue" in record["resource"] else None ) if kind is None: # A reset/shutdown action cannot be safely replayed merely because # its response disappeared. Once no executor can still own the # operation, terminate it visibly instead of allowing a permanent # running/uncertain row to defeat expiry enforcement. return await self._finish_orphan_mutation(record) scoped = components.store job = None job_id = record.get("job_id") if job_id: raw = await asyncio.to_thread(scoped.get, kind, job_id) job = json.loads(raw) if raw else None else: rows = await asyncio.to_thread(scoped.list, kind) for candidate_id, raw in rows.items(): candidate = json.loads(raw) if candidate.get("operation_id") == record["operation_id"]: job_id, job = candidate_id, candidate break if job is None: return await self._finish_orphan_mutation(record) if kind == "snapshot_job" and job_id: state_getter = getattr(app.state, "snapshot_job_state", None) if callable(state_getter): live_job = await asyncio.to_thread(state_getter, job_id) if (isinstance(live_job, dict) and live_job.get("operation_id") == record["operation_id"] and live_job.get("vm_id") == record["vm_uuid"]): if live_job != job: job = live_job try: await asyncio.to_thread( scoped.set, kind, job_id, json.dumps(job, ensure_ascii=False, separators=(",", ":")), ) except Exception: # The in-memory terminal/uncertain state remains # authoritative for this process and will be # offered for persistence again on the next poll. LOG.exception( "Could not retry snapshot job persistence: job=%s", job_id, ) if job.get("owner") != f"php:{record['user_id']}" or job.get("vm_id") != record["vm_uuid"]: return record if kind == "snapshot_job" and job.get("status") == "uncertain": reconciler = getattr(app.state, "reconcile_snapshot_job", None) if callable(reconciler): try: async with self._target_lock( record["host_id"], record["vm_uuid"]): job = await asyncio.to_thread(reconciler, job_id) except Exception: # Reconciliation is evidence-only and never replays the # original mutation. Temporary host/database failures keep # the operation uncertain for a later poll. LOG.exception( "Snapshot reconciliation remains uncertain: op=%s", record["operation_id"], ) record["job_id"] = job_id if kind == "snapshot_job": if isinstance(job.get("result"), dict): record["result"] = dict(job["result"]) else: # Snapshot jobs persist private host evidence (for example # deterministic volume identities) for safe reconciliation. # Never copy the raw job into an operation response. record["result"] = { key: job[key] for key in ( "job_id", "snapshot_key", "action", "consistency", "status", "phase", "created_at", "started_at", "finished_at", ) if key in job } else: record["result"] = { key: value for key, value in job.items() if key not in {"scope", "owner", "operation_id"} } record["result"]["job_id"] = job_id status = job.get("status") record["status"] = ( "succeeded" if status in {"completed", "succeeded"} else "running" if status in {"queued", "running"} else "uncertain" if status == "uncertain" else "failed" ) committed_reinstall = ( kind == "reinstall_job" and _reinstall_committed(job) ) if (committed_reinstall and record["status"] in {"queued", "running", "succeeded", "failed"}): # A progress marker is written after the replacement domain has # started. A later job/result write may be missing, so do not keep # exposing a committed reinstall as running or failed forever. record["status"] = "uncertain" record.pop("finished_at", None) if committed_reinstall and job.get("error_code") not in REINSTALL_PUBLIC_ERROR_CODES: record["error_code"] = ( "reinstall_cleanup_pending" if _reinstall_cleanup_pending(job) else "reinstall_result_uncertain" ) if (("面板重启" in job.get("phase", "") or "面板在任务执行期间重启" in job.get("error", "")) and not committed_reinstall): # The asynchronous job registry explicitly confirms that this job # is no longer running. Replaying a reinstall/rescue mutation is # unsafe, but leaving it uncertain forever would also grant free # service by permanently blocking expiry enforcement. record["status"] = "failed" record["error"] = job.get("error") or "引擎重启导致任务中断,请核查虚拟机状态后重试" if job.get("error"): record["error"] = job["error"] error_code = job.get("error_code") if kind == "snapshot_job" and error_code in SNAPSHOT_PUBLIC_ERROR_CODES: record["error_code"] = error_code if kind == "reinstall_job" and error_code in REINSTALL_PUBLIC_ERROR_CODES: record["error_code"] = error_code if record["status"] in {"succeeded", "failed"}: record["finished_at"] = int(job.get("finished_at") or time.time()) elif record["status"] == "uncertain": record.pop("finished_at", None) await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record)) return record @staticmethod def _validated_delete_cleanup_checkpoint(record: dict) -> dict: """Validate private identifiers before any post-delete cleanup.""" from .libvirt_service import SEED_VOLUME_RE, snapshot_volume_name cleanup_pending = record.get("cleanup_pending") if (cleanup_pending == "vm_delete" and record.get("vm_delete_committed") is not True): raise ValueError("delete commit boundary is not durable") vm_id = record.get("vm_uuid") try: vm_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise ValueError("invalid delete cleanup VM") from exc checkpoint = record.get("delete_cleanup_checkpoint") if checkpoint is None and record.get("cleanup_pending") == "public_ip_claim": # Compatibility for operations durably recorded by releases that # only captured IP claims. Reaching that checkpoint already proved # snapshot, rescue, ownership and credential cleanup completed. checkpoint = { "version": 1, "vm_uuid": vm_id, "snapshots": [], "system_cleanup": False, "legacy_ip_only": True, } allowed_checkpoint_keys = { "version", "vm_uuid", "snapshots", "system_cleanup", "resources", } if cleanup_pending == "public_ip_claim": allowed_checkpoint_keys.add("legacy_ip_only") version = checkpoint.get("version") if isinstance(checkpoint, dict) else None if (not isinstance(checkpoint, dict) or set(checkpoint) - allowed_checkpoint_keys or version not in {1, 2} or checkpoint.get("vm_uuid") != vm_id or not isinstance(checkpoint.get("system_cleanup"), bool)): raise ValueError("invalid delete cleanup checkpoint") resources = checkpoint.get("resources") if version == 1 and resources is not None: raise ValueError("invalid legacy delete resource checkpoint") if version == 2: if (not isinstance(resources, dict) or set(resources) != {"seed_volume", "primary_volume"}): raise ValueError("invalid delete resource checkpoint") seed_volume = resources.get("seed_volume") primary_volume = resources.get("primary_volume") # Reinstall checkpoints carry the exact seed captured before # undefine; reinstall seeds use a fresh UUID unrelated to vm_id. if (seed_volume is not None and (not isinstance(seed_volume, str) or SEED_VOLUME_RE.fullmatch(seed_volume) is None)): raise ValueError("invalid delete seed identity") if (not isinstance(primary_volume, str) or re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9._-]{0,62}\.qcow2", primary_volume, ) is None): raise ValueError("invalid delete primary disk identity") snapshots = checkpoint.get("snapshots") if (not isinstance(snapshots, list) or len(snapshots) > DELETE_CLEANUP_SNAPSHOT_LIMIT): raise ValueError("invalid delete snapshot checkpoint") validated = [] seen = set() for item in snapshots: if not isinstance(item, dict) or set(item) != { "snapshot_key", "volume_name"}: raise ValueError("invalid delete snapshot entry") snapshot_key = item.get("snapshot_key") volume_name = item.get("volume_name") if (not isinstance(snapshot_key, str) or re.fullmatch(r"[0-9a-f]{32}", snapshot_key) is None or snapshot_key in seen or volume_name != snapshot_volume_name(vm_id, snapshot_key)): raise ValueError("invalid deterministic snapshot identity") seen.add(snapshot_key) validated.append({ "snapshot_key": snapshot_key, "volume_name": volume_name, }) keys = record.get("delete_ip_claim_keys") if (not isinstance(keys, list) or len(keys) > 64 or len(set(keys)) != len(keys) or any(not isinstance(key, str) or not key or len(key) > 160 for key in keys)): raise ValueError("invalid delete IP claim checkpoint") validated_checkpoint = { **checkpoint, "vm_uuid": vm_id, "snapshots": validated, "ip_claim_keys": list(keys), } if version == 2: validated_checkpoint["resources"] = { "seed_volume": resources.get("seed_volume"), "primary_volume": resources["primary_volume"], } return validated_checkpoint async def _delete_cleanup_operation(self, record, components): """Finish exact cleanup after VM deletion was durably proven. This method never calls ``delete_vm``. Every destructive identifier is captured before the original child request and validated again here. Missing volumes/rows are accepted so a crash between cleanup steps is safely idempotent. """ if record["operation_id"] in self.active_operations: return record try: checkpoint = self._validated_delete_cleanup_checkpoint(record) except (TypeError, ValueError): record.update( status="uncertain", error="删除任务缺少可信的清理检查点,必须人工核查", ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record profile = components.manager.current() if profile is None or profile.get("key_locked"): return record vm_id = checkpoint["vm_uuid"] scope = self._host_scope(profile) try: expected_snapshots = { item["snapshot_key"]: item["volume_name"] for item in checkpoint["snapshots"] } async with self._target_lock(record["host_id"], vm_id): # Validate all locally persisted identities and claim # ownership while holding the same per-VM lock as ordinary # mutations, before deleting any remaining resource. for key, raw in (await asyncio.to_thread( components.store.list, "vm_snapshot")).items(): try: snapshot = json.loads(raw) except (TypeError, ValueError) as exc: if key.startswith(vm_id + ":"): raise ValueError("damaged snapshot catalog") from exc continue if not isinstance(snapshot, dict): if key.startswith(vm_id + ":"): raise ValueError("damaged snapshot catalog") continue if snapshot.get("vm_id") != vm_id: if key.startswith(vm_id + ":"): raise ValueError("snapshot catalog VM changed") continue snapshot_key = snapshot.get("snapshot_key") if (not isinstance(snapshot_key, str) or key != f"{vm_id}:{snapshot_key}" or expected_snapshots.get(snapshot_key) != snapshot.get("volume_name")): raise ValueError( "snapshot catalog changed after delete") for key in checkpoint["ip_claim_keys"]: raw = await asyncio.to_thread( components.store.get, "ip_claim", key) if raw is None: continue claim = json.loads(raw) if (not isinstance(claim, dict) or claim.get("vm_uuid") != vm_id): raise ValueError("claim ownership changed") service = self._service(profile) resources = checkpoint.get("resources") if resources is not None: await asyncio.to_thread( service.cleanup_deleted_vm_volumes, vm_id, resources) for item in checkpoint["snapshots"]: await asyncio.to_thread( service.delete_snapshot_volume, vm_id, item["snapshot_key"], item["volume_name"], allow_missing_vm=True, ) catalog_key = f"{vm_id}:{item['snapshot_key']}" await asyncio.to_thread( components.store.delete, "vm_snapshot", catalog_key) if await asyncio.to_thread( components.store.get, "vm_snapshot", catalog_key) is not None: raise RuntimeError("snapshot catalog delete not durable") await asyncio.to_thread( service.delete_firewall_filter, vm_id) identity_key = f"{scope}.{vm_id}" await asyncio.to_thread( components.store.delete, "vm_firewall", identity_key) if await asyncio.to_thread( components.store.get, "vm_firewall", identity_key) is not None: raise RuntimeError("firewall record delete not durable") if checkpoint["system_cleanup"]: await asyncio.to_thread( components.store.delete, "vm_rescue", identity_key) if await asyncio.to_thread( components.store.get, "vm_rescue", identity_key) is not None: raise RuntimeError("rescue state delete not durable") await asyncio.to_thread( components.state.users.delete_vm_owner, scope, vm_id) owner_of = getattr(components.state.users, "owner_of_vm", None) if not callable(owner_of): raise RuntimeError("VM owner delete cannot be verified") if await asyncio.to_thread(owner_of, scope, vm_id) is not None: raise RuntimeError("VM owner delete not durable") await asyncio.to_thread( components.store.delete, "vm_credential", identity_key) if await asyncio.to_thread( components.store.get, "vm_credential", identity_key) is not None: raise RuntimeError("credential delete not durable") for key in checkpoint["ip_claim_keys"]: await asyncio.to_thread( components.store.delete, "ip_claim", key) if await asyncio.to_thread( components.store.get, "ip_claim", key) is not None: raise RuntimeError("claim delete not durable") # Never silently leave an uncheckpointed claim for the deleted # VM, and never delete one that was not captured before the VM # mutation. for raw in (await asyncio.to_thread( components.store.list, "ip_claim")).values(): claim = json.loads(raw) if (isinstance(claim, dict) and claim.get("vm_uuid") == vm_id): raise RuntimeError("uncaptured VM claim remains") except Exception: LOG.exception( "VM delete cleanup remains incomplete: op=%s uuid=%s", record.get("operation_id"), vm_id, ) record.update( status="uncertain", error="虚拟机已删除,但快照或控制面清理仍未完成", ) record.pop("finished_at", None) else: record.update( status="succeeded", result={ "uuid": vm_id, "deleted": True, "deleted_snapshots": [ item["snapshot_key"] for item in checkpoint["snapshots"] ], "public_ip_claims_released": True, "firewall_cleaned": True, }, error=None, finished_at=int(time.time()), ) record.pop("error_code", None) record.pop("cleanup_pending", None) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record async def _network_change_operation(self, record, _components): """Keep an ambiguous one-shot public-IP change fail-closed. A single VM metadata read cannot prove that inactive/live XML, the NoCloud seed and clean-traffic filter all converged. It also cannot prove that a partially failed rollback restored every artifact. Before the durable mutation checkpoint, a lost/5xx result is a failed preflight and no target-IP claim exists. After that checkpoint, a single metadata read cannot prove every artifact converged, so both reservations remain unavailable until manual reconciliation. """ if record["operation_id"] in self.active_operations: return record if record.get("network_mutation_started") is False: record.update( status="failed", error="网络变更尚未开始;前置核对被中断", finished_at=int(time.time()), ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record record.update( status="uncertain", error="网络变更结果不确定,已保留新旧公网 IP,必须人工核查", ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record async def _firewall_operation(self, record, components): """Resolve a lost firewall response from exact nwfilter evidence only.""" from .libvirt_service import ( SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS, build_firewall_filter_xml, firewall_policy_digest, ) from .platform_network_policy import ( default_platform_policy_snapshot, normalize_platform_policy_snapshot, ) if record["operation_id"] in self.active_operations: return record target_record = record.get("firewall_target") try: from .app import FirewallPolicyBody if not isinstance(target_record, dict): raise ValueError("firewall target must be an object") if not set(target_record).issubset({ "enabled", "revision", "policy_digest", "rules", "filter_version"}): raise ValueError("firewall target contains unknown fields") target_filter_version = target_record.get("filter_version", 1) if (isinstance(target_filter_version, bool) or not isinstance(target_filter_version, int) or target_filter_version not in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS): raise ValueError("unsupported firewall target compiler") target = FirewallPolicyBody.model_validate({ key: target_record[key] for key in ("enabled", "revision", "policy_digest", "rules") }).model_dump() except Exception: record.update( status="uncertain", error="防火墙操作缺少可信目标策略,必须人工核查", error_code="firewall_result_uncertain", ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record profile = components.manager.current() if profile is None or profile.get("key_locked"): return record if (not record.get("actor_can_manage") or (not record.get("actor_is_admin") and not components.state.users.owns_vm( self._host_scope(profile), record["vm_uuid"], record["user_id"]))): record.update( status="failed", error="虚拟机不存在", error_code="firewall_apply_failed", finished_at=int(time.time()), ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record try: async with self._target_lock(record["host_id"], record["vm_uuid"]): evidence = await asyncio.to_thread( self._service(profile).reconcile_firewall_policy, record["vm_uuid"], enabled=target["enabled"], rules=target["rules"], compiler_version=target_filter_version, ) firewall_key = ( f"{self._host_scope(profile)}.{record['vm_uuid']}") previous_raw = await asyncio.to_thread( components.store.get, "vm_firewall", firewall_key) previous = None previous_valid = previous_raw is None if previous_raw is not None: try: previous = json.loads(previous_raw) previous_policy = FirewallPolicyBody.model_validate({ "enabled": previous["enabled"], "revision": previous["revision"], "policy_digest": previous["policy_digest"], "rules": previous["rules"], }).model_dump() previous_filter_version = previous.get( "filter_version", 1) previous_record_version = previous.get("version") previous_platform_policy = ( normalize_platform_policy_snapshot( previous["platform_policy_snapshot"]) if previous_record_version == 2 else default_platform_policy_snapshot() ) previous_filter_digest = build_firewall_filter_xml( record["vm_uuid"], previous_policy["enabled"], previous_policy["rules"], compiler_version=previous_filter_version, platform_policy_snapshot=previous_platform_policy, )[3] previous_valid = ( previous_record_version in {1, 2} and previous.get("vm_id") == record["vm_uuid"] and not isinstance(previous_filter_version, bool) and isinstance(previous_filter_version, int) and previous_filter_version in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS and not ( previous_record_version == 1 and previous_filter_version >= 4) and ( previous_record_version == 1 or hmac.compare_digest( previous.get( "platform_policy_digest", ""), previous_platform_policy["digest"], ) ) and re.fullmatch( r"[0-9a-f]{64}", previous.get("filter_digest", ""), ) is not None and hmac.compare_digest( previous_policy["policy_digest"], firewall_policy_digest( previous_policy["enabled"], previous_policy["rules"], ), ) and hmac.compare_digest( previous["filter_digest"], previous_filter_digest, ) ) previous = { **previous, **previous_policy, "filter_version": previous_filter_version, "platform_policy_snapshot": previous_platform_policy, "platform_policy_digest": previous_platform_policy["digest"], } except Exception: previous_valid = False observed_policy_digest = evidence.get("policy_digest") digest_matches = ( isinstance(observed_policy_digest, str) and hmac.compare_digest( target["policy_digest"], observed_policy_digest) ) target_can_commit = ( previous_raw is None or ( previous_valid and isinstance(previous, dict) and hmac.compare_digest( previous["platform_policy_digest"], evidence.get("platform_policy_digest", ""), ) and previous["filter_version"] <= target_filter_version and ( previous["revision"] < target["revision"] or ( previous["revision"] == target["revision"] and hmac.compare_digest( previous["policy_digest"], target["policy_digest"], ) ) ) ) ) target_matches = ( evidence.get("matches") is True and digest_matches) if target_matches and target_can_commit: result = { "enabled": target["enabled"], "revision": target["revision"], "policy_digest": evidence["policy_digest"], "filter_digest": evidence["expected_filter_digest"], "filter_version": target_filter_version, "rules": evidence["rules"], "sync_state": "active", } applied_record = { "version": 2, "filter_version": target_filter_version, "vm_id": record["vm_uuid"], **{key: result[key] for key in ( "enabled", "revision", "rules", "policy_digest", "filter_digest", )}, "platform_policy_snapshot": evidence[ "platform_policy_snapshot"], "platform_policy_digest": evidence[ "platform_policy_digest"], "applied_at": int(time.time()), } await asyncio.to_thread( components.store.set, "vm_firewall", firewall_key, json.dumps( applied_record, ensure_ascii=False, separators=(",", ":")), ) else: baseline_old = ( previous_raw is None and evidence.get("coherent") is True and evidence.get("filter_mode") == "baseline" ) observed_filter_digest = evidence.get("filter_digest") recorded_old = ( previous_valid and isinstance(previous, dict) and ( previous["revision"] < target["revision"] or ( previous["revision"] == target["revision"] and previous["filter_version"] < target_filter_version and hmac.compare_digest( previous["policy_digest"], target["policy_digest"], ) ) ) and evidence.get("coherent") is True and evidence.get("filter_mode") == "managed" and isinstance(observed_filter_digest, str) and hmac.compare_digest( observed_filter_digest, previous["filter_digest"], ) ) # Only an intact baseline or the exact prior persisted # filter proves that the attempted target did not commit. # Any partial/mixed/unreadable evidence remains uncertain. result = ( None if not target_matches and (baseline_old or recorded_old) else ... ) except Exception: LOG.exception( "Firewall reconciliation remains uncertain: op=%s", record.get("operation_id"), ) result = ... if isinstance(result, dict): record.update( status="succeeded", result=result, error=None, finished_at=int(time.time()), ) record.pop("error_code", None) elif result is None: record.update( status="failed", error="引擎证据确认目标防火墙策略未生效", error_code="firewall_apply_failed", finished_at=int(time.time()), ) else: record.update( status="uncertain", error="防火墙生效证据不完整,必须继续核对原操作", error_code="firewall_result_uncertain", ) record.pop("finished_at", None) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record async def _uncertain_delete_operation(self, record): """Keep a lost destructive delete bound to its original operation. A missing executor does not prove whether the domain definition and every panel-managed volume were removed. Converting this state to a terminal failure would let PHP issue a second DELETE with a new operation identifier. Only the explicit claim-cleanup path above may finish automatically after the child endpoint durably proved that VM deletion itself completed. """ if record["operation_id"] in self.active_operations: return record record.pop("finished_at", None) record.update( status="uncertain", error="删除结果不确定,必须继续核对原操作,不能重复删除", ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record async def _power_action_operation(self, record, components): """Reconcile a lost synchronous power request by observation only. A power command is never replayed after its executor disappears. The engine action itself verifies the postcondition before returning, and this path covers the remaining transport/process-loss window by reading the lightweight VM state endpoint. A transitional or mismatched state remains visible, while reboot/reset stay uncertain when the final state cannot prove that the edge-triggered command committed. """ if (record["operation_id"] in self.active_operations or int(time.time()) - int(record.get("created_at", 0)) < 120): return record action = record.get("target_action") if action not in POWER_ACTIONS: # Rows written before target_action was introduced retain the old # no-replay behavior; do not infer an action from arbitrary body # data during recovery. return await self._finish_orphan_mutation(record) if (record.get("actor_can_manage") is not True and record.get("actor_is_admin") is not True): return await self._finish_orphan_mutation(record) profile = await asyncio.to_thread(components.manager.current) if profile is None or profile.get("key_locked"): return record if not record.get("actor_is_admin"): try: owns = await asyncio.to_thread( components.state.users.owns_vm, self._host_scope(profile), record["vm_uuid"], record["user_id"], ) except Exception: return record if not owns: return await self._finish_orphan_mutation(record) service = self._service(profile) getter = getattr(service, "get_vm_state", None) if not callable(getter): # Compatibility with a short-lived older service implementation; # get_vm still returns the same bounded state field, at a higher # cost. Production VMService always has get_vm_state(). getter = getattr(service, "get_vm", None) if not callable(getter): return record from .libvirt_service import VMError try: # Keep the observation, conclusion, and durable operation write in # the same per-VM critical section as every other mutation. A new # action must not race this recovery read and make an otherwise # matching state stale immediately. async with self._target_lock(record["host_id"], record["vm_uuid"]): try: observed = await asyncio.to_thread( getter, record["vm_uuid"]) except Exception as exc: if isinstance(exc, VMError) and exc.status == 404: record.update( status="failed", result=None, error="虚拟机不存在", finished_at=int(time.time()), ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record), ) # A transport/read failure is not evidence of either # outcome; leave the record available for a later # read-only reconciliation. return record if not isinstance(observed, dict): return record if observed.get("uuid") != record["vm_uuid"]: return record state = observed.get("state") if action in POWER_ACTION_EDGE_ACTIONS: # reset always leaves the domain running, and a normal # reboot usually does too. A reboot that leaves the # domain shut off may be valid under on_reboot=destroy, # but this durable row does not contain that XML policy. # No observed state can prove either edge-triggered # command committed, so keep it uncertain even when the # domain is paused, crashed, or in transition. record.update( status="uncertain", result=None, error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态", ) record.pop("finished_at", None) elif state not in POWER_ACTION_TERMINAL_STATES[action]: record.update( status="failed", result=None, error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态", finished_at=int(time.time()), ) else: result = { key: observed[key] for key in ("uuid", "name", "state") if key in observed } record.update( status="succeeded", result=result, error=None, finished_at=int(time.time()), safe_to_refund=False, ) record.pop("error_code", None) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record except Exception: # A lock/store failure is not evidence of the VM's power state; # leave the durable uncertainty available for a later pass. LOG.exception( "Power action reconciliation remains unresolved: op=%s", record.get("operation_id"), ) return record async def _finish_orphan_mutation(self, record): if (record["operation_id"] in self.active_operations or int(time.time()) - int(record.get("created_at", 0)) < 120): return record record.update( status="failed", error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态", finished_at=int(time.time()), ) await asyncio.to_thread( self.store.set, "operation", record["operation_id"], json.dumps(record)) return record @staticmethod def _service_target_method(resource: str) -> str | None: match = re.fullmatch( rf"vms/{VM_ID}/((?:service|network)-(?:suspend|resume))", resource) if match is None: return None return { "service-suspend": "service_suspend", "service-resume": "service_resume", "network-suspend": "network_suspend", "network-resume": "network_resume", }[match[1]] @staticmethod def _is_network_change_resource(resource: str) -> bool: return re.fullmatch(rf"vms/{VM_ID}/network", resource) is not None @staticmethod def _is_power_action_resource(resource: str) -> bool: return re.fullmatch(rf"vms/{VM_ID}/actions", resource) is not None @staticmethod def _is_firewall_resource(resource: str) -> bool: return re.fullmatch(rf"vms/{VM_ID}/firewall", resource) is not None @staticmethod def _is_vm_delete_operation(operation: dict) -> bool: return (operation.get("method") == "DELETE" and re.fullmatch( rf"vms/{VM_ID}", operation.get("resource", "")) is not None) @staticmethod def _host_scope(profile): from .metrics import host_scope return host_scope(profile) async def _capture_service_was_running_hint(self, host_id, operation, delegated): # Run this inside the same per-VM lock as service_suspend. If the # engine crashes after stopping QEMU but before saving the response, a # replay otherwise sees only a shutoff guest and cannot know whether a # later renewal is allowed to start it automatically. components = self.registry.components[host_id] profile = components.manager.current() owns = (profile is not None and not profile.get("key_locked") and (delegated.get("is_admin") or components.state.users.owns_vm( self._host_scope(profile), operation["vm_uuid"], delegated["user_id"]))) if not owns: return before = await asyncio.to_thread( self._service(profile).get_vm, operation["vm_uuid"]) operation["service_was_running_hint"] = before.get("state") == "running" # This write is a prerequisite for stopping the guest. Propagate any # failure so the child endpoint is never called without a durable hint. await asyncio.to_thread( self.store.set, "operation", operation["operation_id"], json.dumps(operation)) async def _capture_network_transition(self, host_id, operation): """Persist the source address before invoking the one-shot mutation.""" components = self.registry.components[host_id] profile = components.manager.current() if profile is None or profile.get("key_locked"): raise EngineAuthError( "宿主机未解锁,无法保存网络变更前状态", 409) target = operation.get("network_target_cidr") before = await asyncio.to_thread( self._service(profile).get_vm, operation["vm_uuid"]) source = before.get("public_ip_cidr") \ if isinstance(before, dict) else None if not isinstance(source, str) or not isinstance(target, str): raise EngineAuthError( "无法保存网络变更前后的公网 IP,已取消执行", 409) operation["network_source_cidr"] = source operation["network_mutation_started"] = False source_pool_key = before.get("public_pool_key") \ if isinstance(before, dict) else None if (isinstance(source_pool_key, str) and (source_pool_key == "legacy-default" or re.fullmatch(r"[0-9a-f]{32}", source_pool_key))): operation["network_source_pool_key"] = source_pool_key # This durable write is a prerequisite for calling the child endpoint. # If it fails, the exception prevents libvirt from being touched. await asyncio.to_thread( self.store.set, "operation", operation["operation_id"], json.dumps(operation)) async def _capture_delete_cleanup(self, host_id, operation, delegated): """Persist exact private cleanup identities before deleting the VM.""" from .libvirt_service import snapshot_volume_name components = self.registry.components[host_id] vm_id = str(uuid.UUID(operation["vm_uuid"])) snapshots = [] for key, raw in (await asyncio.to_thread( components.store.list, "vm_snapshot")).items(): try: snapshot = json.loads(raw) except (TypeError, ValueError) as exc: if key.startswith(vm_id + ":"): raise EngineAuthError( "虚拟机快照目录损坏,已取消删除", 409) from exc continue if not isinstance(snapshot, dict): if key.startswith(vm_id + ":"): raise EngineAuthError( "虚拟机快照目录损坏,已取消删除", 409) continue if snapshot.get("vm_id") != vm_id: if key.startswith(vm_id + ":"): raise EngineAuthError( "虚拟机快照目录身份不一致,已取消删除", 409) continue snapshot_key = snapshot.get("snapshot_key") volume_name = snapshot.get("volume_name") if (not isinstance(snapshot_key, str) or re.fullmatch(r"[0-9a-f]{32}", snapshot_key) is None or key != f"{vm_id}:{snapshot_key}" or volume_name != snapshot_volume_name(vm_id, snapshot_key)): raise EngineAuthError( "虚拟机快照目录身份不一致,已取消删除", 409) snapshots.append({ "snapshot_key": snapshot_key, "volume_name": volume_name, }) # Current policy retains at most two usable recovery points, but old # confirmed-failed catalog rows may predate automatic pruning. Keep a # bounded compatibility window so expiry deletion can safely clean # those deterministic volumes instead of becoming permanently stuck. if len(snapshots) > DELETE_CLEANUP_SNAPSHOT_LIMIT or len({ item["snapshot_key"] for item in snapshots}) != len(snapshots): raise EngineAuthError("虚拟机快照目录数量或身份无效,已取消删除", 409) keys = [] for key, raw in (await asyncio.to_thread( components.store.list, "ip_claim")).items(): try: claim = json.loads(raw) except (TypeError, ValueError): continue if (isinstance(claim, dict) and claim.get("vm_uuid") == operation["vm_uuid"]): keys.append(key) operation["delete_ip_claim_keys"] = sorted(keys) profile = await asyncio.to_thread(components.manager.current) resources = None if (operation.get("delete_disks") is True and profile is not None and not profile.get("key_locked")): service = self._service(profile) capture = getattr(service, "delete_vm_cleanup_manifest", None) if callable(capture): resources = await asyncio.to_thread(capture, vm_id) if (not isinstance(resources, dict) or resources.get("vm_uuid") != vm_id): raise EngineAuthError("虚拟机删除检查点无效,已取消删除", 409) resources = { "seed_volume": resources.get("seed_volume"), "primary_volume": resources.get("primary_volume"), } checkpoint = { "version": 2 if resources is not None else 1, "vm_uuid": vm_id, "snapshots": sorted( snapshots, key=lambda item: item["snapshot_key"]), "system_cleanup": delegated.get("system_cleanup") is True, } if resources is not None: checkpoint["resources"] = resources operation["delete_cleanup_checkpoint"] = checkpoint # The complete private checkpoint is durable before libvirt or storage # deletion begins. A write failure aborts the child call rather than # losing exact cleanup proof. await asyncio.to_thread( self.store.set, "operation", operation["operation_id"], json.dumps(operation)) async def _check_absent(self, operation): operation["safe_to_refund"] = False components = self.registry.components.get(operation["host_id"]) profile = components.manager.current() if components else None if profile and not profile.get("key_locked"): try: absent = await asyncio.to_thread(self._service(profile).provisioning_absent, operation["vm_uuid"], operation["vm_name"], uuid_resources=operation.get("resource_naming") == "uuid-v1", legacy_name_collision=( operation.get("resource_naming") is None and operation.get("error") == "虚拟机名称已存在" )) if absent: await asyncio.to_thread(self._cleanup_absent_records, components, profile, operation["vm_uuid"]) operation.update(status="failed", safe_to_refund=True) except Exception: pass @staticmethod def _cleanup_absent_records(components, profile, vm_id): from .metrics import host_scope scope = host_scope(profile) components.state.users.delete_vm_owner(scope, vm_id) components.store.delete("vm_credential", f"{scope}.{vm_id}") prefix = scope + "." for key, raw in components.store.list("ip_claim").items(): if key.startswith(prefix) and json.loads(raw).get("vm_uuid") == vm_id: components.store.delete("ip_claim", key) def _public_operation(self, record): result = {key: record[key] for key in ( "operation_id", "status", "result", "error", "safe_to_refund", "service_was_running_hint", "created_at", "finished_at") if key in record} if ("/snapshots" in record.get("resource", "") and record.get("error_code") in SNAPSHOT_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] if (record.get("resource", "").endswith("/reinstall") and record.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] if (self._is_firewall_resource(record.get("resource", "")) and record.get("error_code") in FIREWALL_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] if (self._is_vm_delete_operation(record) and record.get("error_code") in DELETE_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] if (self._is_network_change_resource(record.get("resource", "")) and record.get("network_mutation_started") is False and record.get("error_code") in FIREWALL_PUBLIC_ERROR_CODES): result["error_code"] = record["error_code"] # This is deliberately live process state, not a persisted claim. PHP # must not close a stale policy job while its original executor can # still mutate libvirt. It retries after the executor becomes terminal. result["executor_active"] = record.get("operation_id") in self.active_operations return result @staticmethod def _service(profile): from .libvirt_service import VMService return VMService(profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"], profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", "")) async def _ticket(self, host_id, vm_id, delegated, send): if not delegated.get("session_id"): raise EngineAuthError("控制台需要有效的网站会话", 403) components = self.registry.components[host_id] profile = components.manager.current() if profile is None or profile.get("key_locked"): raise EngineAuthError("宿主机尚未就绪", 409) from .metrics import host_scope if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]): raise EngineAuthError("虚拟机不存在", 404) await asyncio.to_thread(self._service(profile).console_port, vm_id) if not await self._session_active(delegated, host_id, vm_id): raise EngineAuthError("网站会话已失效", 403) ticket = secrets.token_urlsafe(32) key = hashlib.sha256(ticket.encode("ascii")).hexdigest() record = {"host_id": host_id, "vm_id": vm_id, "actor": delegated, "expires_at": int(time.time()) + 60} await asyncio.to_thread(self.store.insert, "console_ticket", key, json.dumps(record)) await json_response(send, 200, {"ticket": ticket, "expires_in": 60, "websocket_path": "/engine-console"}) async def _session_active(self, delegated, host_id, vm_id): if self.introspection is not None: return await self.introspection(delegated, host_id, vm_id) return await asyncio.to_thread(self._introspect, delegated, host_id, vm_id) def _introspect(self, delegated, host_id, vm_id): url = self.config.get("php_introspection_url", "") parsed = urlsplit(url) if (parsed.scheme not in {"http", "https"} or parsed.hostname not in {"127.0.0.1", "::1", "localhost"} or parsed.username or parsed.password or parsed.fragment): return False body = json.dumps({"session_id": delegated["session_id"], "user_id": delegated["user_id"], "host_id": host_id, "vm_uuid": vm_id}, separators=(",", ":")).encode() secret = decode_secret(self.config.get("introspection_secret", self.config["service_secret"])) headers = signed_headers(secret, "POST", parsed.path + ("?" + parsed.query if parsed.query else ""), body, delegated) headers["Content-Type"] = "application/json" if self.config.get("php_introspection_host"): headers["Host"] = self.config["php_introspection_host"] try: opener = build_opener(ProxyHandler({}), NoRedirects()) with opener.open(URLRequest(url, body, headers, method="POST"), timeout=5) as response: raw = response.read(8193) return response.status == 200 and len(raw) <= 8192 and json.loads(raw).get("active") is True except Exception: return False async def _console(self, scope, receive, send): from starlette.websockets import WebSocket socket = WebSocket(scope, receive=receive, send=send) if scope["path"] != "/engine-console" or scope.get("scheme") != "wss": await socket.close(code=4403) return from .app import _origin_matches_host if not _origin_matches_host(socket.headers.get("origin"), socket.headers.get("host")): await socket.close(code=4403) return protocols = [value for value in scope.get("subprotocols", []) if value.startswith("cloud-ticket.")] if len(protocols) != 1 or not re.fullmatch(r"cloud-ticket\.[A-Za-z0-9_-]{43}", protocols[0]): await socket.close(code=4401) return key = hashlib.sha256(protocols[0][13:].encode("ascii")).hexdigest() raw = await asyncio.to_thread(self.store.get, "console_ticket", key) if raw is None: await socket.close(code=4401) return record = json.loads(raw) if record["expires_at"] < int(time.time()) or not await asyncio.to_thread(self.store.insert, "ticket_used", key, "1"): await socket.close(code=4401) return await asyncio.to_thread(self.store.delete, "console_ticket", key) delegated, host_id, vm_id = record["actor"], record["host_id"], record["vm_id"] if not await self._session_active(delegated, host_id, vm_id): await socket.close(code=4403) return token = TRUSTED_ACTOR.set(delegated) relay = watcher = None try: await self.registry.get(host_id) components = self.registry.components[host_id] profile = components.manager.current() from .metrics import host_scope from .console import relay_console if profile is None or profile.get("key_locked"): await socket.close(code=4409) return if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]): await socket.close(code=4403) return port = await asyncio.to_thread(self._service(profile).console_port, vm_id) await socket.accept(subprotocol=protocols[0]) async def watch(): while True: await asyncio.sleep(self.console_check_interval) if not await self._session_active(delegated, host_id, vm_id): return current = components.manager.current() if current is None or current.get("key_locked"): return if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(current), vm_id, delegated["user_id"]): return relay = asyncio.create_task(relay_console(socket, port=port, ssh_target=profile["ssh_target"], ssh_binary="ssh", known_hosts_file=profile["ssh_known_hosts_file"], ssh_config_file=profile["ssh_config_file"])) watcher = asyncio.create_task(watch()) await asyncio.wait({relay, watcher}, return_when=asyncio.FIRST_COMPLETED) except Exception: LOG.warning("Engine console closed after transport/authorization failure") finally: for task in (relay, watcher): if task is not None: task.cancel() await asyncio.gather(*(task for task in (relay, watcher) if task is not None), return_exceptions=True) with suppress(Exception): await socket.close(code=4403) TRUSTED_ACTOR.reset(token) async def _lifespan(self, receive, send): while True: message = await receive() if message["type"] == "lifespan.startup": try: await self.registry.start() # The supervisor is fed only by durable tombstones created # by an explicit delete. It never makes reads or unrelated # writes perform GC, pauses during online update, and keeps # retrying offline/locked hosts after they recover. self.cache_cleanup_event = asyncio.Event() self.cache_cleanup_supervisor = asyncio.create_task( self._cache_cleanup_loop()) self.cache_cleanup_event.set() await send({"type": "lifespan.startup.complete"}) except Exception: LOG.exception("Engine startup failed") await send({"type": "lifespan.startup.failed", "message": "引擎初始化失败"}) return elif message["type"] == "lifespan.shutdown": supervisor = self.cache_cleanup_supervisor if supervisor is not None: supervisor.cancel() await asyncio.gather(supervisor, return_exceptions=True) self.cache_cleanup_supervisor = None self.cache_cleanup_event = None # A task awaiting asyncio.to_thread cannot stop the underlying # libvirt call when cancelled. Drain these bounded cleanup # tasks before closing host services so no hidden mutation can # outlive the engine lifespan. await asyncio.gather( *tuple(self.maintenance_tasks), return_exceptions=True) await self.registry.close() await send({"type": "lifespan.shutdown.complete"}) return def create_engine(): default = Path(__file__).resolve().parents[2] / "runtime" / "engine.json" config = private_config(Path(os.environ.get("CLOUD_ENGINE_CONFIG", str(default)))) secret = decode_secret(config["service_secret"]) del secret key = master_key(Path(config["master_key_file"])) store = MainStore(config["main_database"], key) logs = LogStore(config["log_database"], key) return EngineApplication(config, store, logs)