You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/internal.py

1915 lines
95 KiB
Python

"""Private PHP -> libvirt boundary. The legacy browser app is never exposed.
Only allowlisted VM/host/template resources are dispatched to per-host services.
The public console endpoint accepts a short-lived, one-use ticket instead of a
PHP cookie, and rechecks the PHP session while the console remains connected.
"""
from __future__ import annotations
import asyncio
import errno
import hashlib
import hmac
import inspect
import json
import logging
import os
import re
import secrets
import stat
import subprocess
import tempfile
import threading
import time
import uuid
from contextlib import AsyncExitStack, suppress
from pathlib import Path
from urllib.parse import parse_qsl, urlsplit
from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLRequest,
build_opener)
from .cloud_templates import (
MAX_DOWNLOAD_SECONDS,
MAX_IMAGE_BYTES,
TemplateError,
template_volume_name,
validate_template_metadata,
validate_template_name,
verify_qcow2_image,
)
from .cloud_template_runtime import (
CACHE_TOMBSTONE_KIND,
MIN_DOWNLOAD_BYTES,
CloudTemplateRuntime,
)
from .engine_context import (EngineComponents, HostStore, LogStore, MainStore,
TRUSTED_ACTOR, actor_session)
from .engine_security import (EngineAuthError, authenticate, decode_secret,
signed_headers)
from .updater import UpdateError, Updater
from .user_isos import UserISORuntime
LOG = logging.getLogger(__name__)
HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
OP_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z")
VM_ID = r"[0-9a-fA-F-]{36}"
JSON_LIMIT = 1024 * 1024
CLOUD_UPLOAD_IDLE_SECONDS = 60
CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024
READ_PATHS = (
r"host", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms",
r"vm-states", r"vm-metadata",
r"user-isos/[0-9a-f]{32}",
rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue)?",
r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+",
)
WRITE_PATHS = {
"POST": (r"host/(?:scan|unlock|test|discover-vms)",
r"vm-templates/(?:import|upload)", r"rescue-templates",
r"user-isos/import",
r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|rescue|console-ticket)",
rf"vms/{VM_ID}/(?:service|network)-(?:suspend|resume)",
rf"vms/{VM_ID}/rescue/(?:exit|force-start)", r"reinstall-jobs/[A-Za-z0-9_-]+/ack-password"),
"PUT": (r"host",),
"PATCH": (r"vm-templates/[A-Za-z0-9_-]+",),
"DELETE": (rf"vms/{VM_ID}", r"vm-templates/[A-Za-z0-9_-]+",
r"rescue-templates/[A-Za-z0-9_-]+",
r"user-isos/[0-9a-f]{32}"),
}
class _CloudUploadWriter:
"""Own a temporary upload descriptor used only from worker threads."""
def __init__(self, directory: Path) -> None:
descriptor, raw_path = tempfile.mkstemp(
prefix=".cloud-image-upload-",
suffix=".qcow2",
dir=directory,
)
try:
os.fchmod(descriptor, 0o600)
except BaseException:
os.close(descriptor)
Path(raw_path).unlink(missing_ok=True)
raise
self.path = Path(raw_path)
self._descriptor = descriptor
self._lock = threading.Lock()
def write(self, chunk: memoryview) -> None:
"""Write the complete bounded chunk before accepting another one."""
with self._lock:
if self._descriptor < 0:
raise OSError(errno.EBADF, "cloud upload is closed")
remaining = chunk
while remaining:
written = os.write(self._descriptor, remaining)
if written <= 0:
raise OSError(errno.EIO, "cloud upload write made no progress")
remaining = remaining[written:]
def sync_and_close(self) -> None:
"""Durably flush and close the descriptor before image validation."""
with self._lock:
if self._descriptor < 0:
return
descriptor = self._descriptor
try:
os.fsync(descriptor)
finally:
try:
os.close(descriptor)
finally:
self._descriptor = -1
def close(self) -> None:
"""Close after an interrupted request, waiting for an in-flight write."""
with self._lock:
if self._descriptor < 0:
return
descriptor = self._descriptor
try:
os.close(descriptor)
finally:
self._descriptor = -1
class NoRedirects(HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
raise ValueError("Session introspection redirects are not allowed")
class UpdateServiceUncertain(UpdateError):
"""The worker could not be confirmed stopped, so its job must stay active."""
def private_config(path: Path) -> dict:
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 or info.st_uid != os.geteuid():
raise ValueError("引擎配置必须为运行用户拥有的普通 0600 文件")
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
raw = os.read(descriptor, JSON_LIMIT + 1)
finally:
os.close(descriptor)
if len(raw) > JSON_LIMIT:
raise ValueError("引擎配置过大")
value = json.loads(raw)
if not isinstance(value, dict):
raise ValueError("引擎配置无效")
return value
def master_key(path: Path) -> bytes:
from .database_bootstrap import DatabaseBootstrap
if not path.is_absolute():
raise ValueError("主密钥路径必须是绝对路径")
bootstrap = DatabaseBootstrap(path.parent)
bootstrap.key_file = path
return bootstrap.master_key(create=True)
class HostRegistry:
def __init__(self, config, store, log_store, *, factory=None, user_isos=None,
cloud_templates=None):
self.config = config
self.store = store
self.log_store = log_store
self.factory = factory
self.user_isos = user_isos or UserISORuntime(
store, Path(config["data_dir"]))
self._owns_user_isos = user_isos is None
self.cloud_templates = cloud_templates or CloudTemplateRuntime(
store, Path(config["data_dir"]))
self._owns_cloud_templates = cloud_templates is None
self.components: dict[str, EngineComponents] = {}
self.apps = {}
# Constructed by synchronous factories before an event loop may exist.
# Bind the lock lazily inside the ASGI lifespan/request loop.
self.lock: asyncio.Lock | None = None
self.lifespans = AsyncExitStack()
async def start(self):
if self.lock is None:
self.lock = asyncio.Lock()
await asyncio.to_thread(self.user_isos.restore_jobs)
await asyncio.to_thread(self.cloud_templates.restore_jobs)
await self.lifespans.__aenter__()
for host_id in await asyncio.to_thread(self.store.list, "engine_host"):
await self.get(host_id)
async def close(self):
await self.lifespans.aclose()
if self._owns_user_isos:
await asyncio.to_thread(self.user_isos.close)
if self._owns_cloud_templates:
await asyncio.to_thread(self.cloud_templates.close)
async def get(self, host_id: str):
if not HOST_ID.fullmatch(host_id):
raise EngineAuthError("宿主机标识无效", 400)
if self.lock is None:
self.lock = asyncio.Lock()
async with self.lock:
if host_id in self.apps:
return self.apps[host_id]
from .app import create_app
from .config import Settings
directory = Path(self.config["data_dir"]) / "hosts" / host_id
if self.factory is not None:
try:
inspect.signature(self.factory).bind(
host_id, self.user_isos, self.cloud_templates)
except TypeError:
try:
inspect.signature(self.factory).bind(host_id, self.user_isos)
except TypeError:
components, app = self.factory(host_id)
else:
components, app = self.factory(host_id, self.user_isos)
else:
components, app = self.factory(
host_id, self.user_isos, self.cloud_templates)
else:
components = await asyncio.to_thread(EngineComponents.for_host,
self.store, self.log_store, host_id, directory,
self.user_isos, self.cloud_templates)
settings = Settings(data_dir=directory)
app = await asyncio.to_thread(create_app, settings, engine_components=components)
self.components[host_id] = components
await self.lifespans.enter_async_context(app.router.lifespan_context(app))
self.apps[host_id] = app
return app
async def json_response(send, status: int, payload):
body = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode()
await send({"type": "http.response.start", "status": status,
"headers": [(b"content-type", b"application/json"), (b"cache-control", b"no-store")]})
await send({"type": "http.response.body", "body": body})
def raw_target(scope) -> str:
path = scope.get("raw_path", scope["path"].encode("ascii")).decode("ascii")
query = scope.get("query_string", b"")
return path + ("?" + query.decode("ascii") if query else "")
def request_headers(scope):
values = {}
for key, value in scope.get("headers", []):
name = key.decode("ascii").lower()
if name.startswith("x-cloud-") and name in values:
raise EngineAuthError("重复的内部认证头")
values[name] = value.decode("latin-1")
return values
class EngineApplication:
def __init__(self, config, store, log_store, *, registry=None, introspection=None,
updater=None, update_trigger=None):
self.config = config
self.store = store
self.secret = decode_secret(config["service_secret"])
self.registry = registry or HostRegistry(config, store, log_store)
self.introspection = introspection
project_dir = Path(config.get("project_dir", Path(__file__).resolve().parents[2]))
self.updater = updater or Updater(
project_dir, store, worker_active=self._update_service_active)
self.update_trigger = update_trigger or self._start_update_service
# Serializes the transition into update maintenance mode with every
# request that can mutate libvirt or durable operation state. The
# lock is created lazily so synchronous test/application factories do
# not bind it to an event loop that has not started yet.
self.update_coordination_lock: asyncio.Lock | None = None
self.active_operations: set[str] = set()
self.maintenance_tasks: set[asyncio.Task] = set()
# Target-state operations are idempotent individually, but two
# different operation IDs can request opposite states at the same
# time (for example traffic suspension ending at a period boundary).
# Serialize those mutations per VM so the later request always wins
# on the hypervisor as well as in the billing database.
self.target_locks: dict[tuple[str, str], asyncio.Lock] = {}
self.console_check_interval = 15
def _background(self, coroutine) -> None:
task = asyncio.create_task(coroutine)
self.maintenance_tasks.add(task)
task.add_done_callback(self.maintenance_tasks.discard)
def _target_lock(self, host_id: str, vm_id: str) -> asyncio.Lock:
key = (host_id, vm_id)
lock = self.target_locks.get(key)
if lock is None:
lock = asyncio.Lock()
self.target_locks[key] = lock
return lock
async def __call__(self, scope, receive, send):
if scope["type"] == "lifespan":
await self._lifespan(receive, send)
return
if scope["type"] == "websocket":
try:
await self._console(scope, receive, send)
except Exception:
LOG.warning("Engine console rejected because a dependency failed")
await send({"type": "websocket.close", "code": 4403})
return
if scope["type"] != "http":
return
token = None
response_started = False
async def tracked_send(message):
nonlocal response_started
if message["type"] == "http.response.start":
response_started = True
await send(message)
try:
# Public reverse proxies must never forward these routes. Reject
# non-loopback peers even when they have spoofed forwarded headers.
client = scope.get("client")
if client and client[0] not in {"127.0.0.1", "::1", "testclient"}:
raise EngineAuthError("内部引擎仅接受本机服务请求", 403)
headers = request_headers(scope)
target = raw_target(scope)
upload_resource = None
if scope["method"] == "POST":
if scope["path"] == "/internal/v1/vm-templates/upload":
upload_resource = "vm-templates/upload"
elif re.fullmatch(
r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates",
scope["path"]):
upload_resource = "rescue-templates"
upload = upload_resource is not None
if upload:
digest = headers.get("x-cloud-content-sha256", "")
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
scope["method"], target, digest, self.store)
self._authorize_route(
scope["method"], upload_resource, delegated)
checksum = hashlib.sha256()
validation = {"complete": False, "valid": False}
scope = {
**scope,
"engine_upload_validation": validation,
"engine_upload_digest": digest,
}
async def verified_receive():
message = await receive()
if message["type"] == "http.request":
chunk = message.get("body", b"")
checksum.update(chunk)
if not message.get("more_body", False):
validation.update(complete=True,
valid=hmac.compare_digest(checksum.hexdigest(), digest))
return message
child_receive = verified_receive
body = None
else:
chunks = []
size = 0
while True:
message = await receive()
if message["type"] == "http.disconnect":
return
chunk = message.get("body", b"")
size += len(chunk)
if size > JSON_LIMIT:
raise EngineAuthError("内部请求超过 1 MiB", 413)
chunks.append(chunk)
if not message.get("more_body", False):
break
body = b"".join(chunks)
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
scope["method"], target, hashlib.sha256(body).hexdigest(), self.store)
delivered = False
async def replay_receive():
nonlocal delivered
if not delivered:
delivered = True
return {"type": "http.request", "body": body, "more_body": False}
return await receive()
child_receive = replay_receive
token = TRUSTED_ACTOR.set(delegated)
await self._dispatch(scope, child_receive, tracked_send, body, delegated)
except EngineAuthError as exc:
if not response_started:
await json_response(send, exc.status, {"error": str(exc)})
except UpdateError as exc:
if not response_started:
await json_response(send, exc.status, {"error": str(exc)})
except Exception:
LOG.exception("Private engine request failed")
if not response_started:
await json_response(send, 503, {"error": "引擎服务暂时不可用,请检查引擎日志"})
finally:
if token is not None:
TRUSTED_ACTOR.reset(token)
@staticmethod
def _authorize_route(method: str, resource: str, delegated: dict):
paths = READ_PATHS if method == "GET" else WRITE_PATHS.get(method, ())
if not any(re.fullmatch(pattern, resource) for pattern in paths):
raise EngineAuthError("内部资源不存在", 404)
administration = (resource.startswith("host") or resource.startswith("vm-templates/")
or resource.startswith("rescue-templates/"))
administration_write = method != "GET" and (administration or resource == "rescue-templates")
if administration and not delegated["is_admin"]:
raise EngineAuthError("仅管理员可访问此资源", 403)
if method != "GET" and not delegated["can_manage"]:
raise EngineAuthError("当前账户不允许操作", 403)
if administration_write and not delegated["is_super_admin"]:
raise EngineAuthError("仅超级管理员可修改宿主机和模版", 403)
async def _dispatch(self, scope, receive, send, body, delegated):
path = scope["path"]
if path.startswith("/internal/v1/update/"):
await self._update_dispatch(scope["method"], path, body, delegated, send)
return
method = scope["method"]
operation_reconcile = (
method == "GET"
and re.fullmatch(r"/internal/v1/operations/[A-Za-z0-9_-]{8,80}", path) is not None
and scope.get("query_string", b"") != b"reconcile=false"
)
# A browser upload can spend minutes waiting for request-body chunks.
# Its handler acquires the update coordination lock only to reserve and
# finally queue the import, so unrelated VM writes remain available.
if method == "POST" and path == "/internal/v1/vm-templates/upload":
await self._dispatch_regular(scope, receive, send, body, delegated)
return
if method != "GET" or operation_reconcile:
async with self._update_lock():
if await asyncio.to_thread(self.updater.active_job):
raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409)
await self._dispatch_regular(scope, receive, send, body, delegated)
return
await self._dispatch_regular(scope, receive, send, body, delegated)
def _update_lock(self) -> asyncio.Lock:
if self.update_coordination_lock is None:
self.update_coordination_lock = asyncio.Lock()
return self.update_coordination_lock
async def _dispatch_regular(self, scope, receive, send, body, delegated):
path = scope["path"]
if scope["method"] == "POST" and path == "/internal/v1/hosts/scan":
if not delegated["is_super_admin"] or not delegated["can_manage"]:
raise EngineAuthError("仅超级管理员可以扫描宿主机", 403)
from .app import HostScanBody
from .remote_host import RemoteHostManager
from pydantic import ValidationError
try:
value = HostScanBody.model_validate_json(body)
except ValidationError as exc:
raise EngineAuthError("宿主机扫描参数无效", 422) from exc
manager = await asyncio.to_thread(RemoteHostManager,
Path(self.config["data_dir"]) / "discovery")
try:
result = await asyncio.to_thread(manager.scan, value.host, value.port)
await json_response(send, 200, result)
finally:
await asyncio.to_thread(manager.close)
return
if scope["method"] == "GET" and path == "/internal/v1/health":
await asyncio.to_thread(self.store.get, "engine", "readiness")
await json_response(send, 200, {"status": "ok"})
return
cloud_template_match = re.fullmatch(
r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|"
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
if cloud_template_match is not None:
resource = cloud_template_match[1]
self._authorize_route(scope["method"], resource, delegated)
await self._cloud_template_dispatch(
scope["method"], resource, body, send,
scope=scope, receive=receive)
return
user_iso_match = re.fullmatch(
r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|"
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
if user_iso_match is not None:
resource = user_iso_match[1]
self._authorize_route(scope["method"], resource, delegated)
await self._user_iso_dispatch(
scope["method"], resource, body, delegated, send)
return
op_match = re.fullmatch(r"/internal/v1/operations/([A-Za-z0-9_-]{8,80})", path)
if scope["method"] == "GET" and op_match:
query = scope.get("query_string", b"")
if query not in {b"", b"reconcile=false"}:
raise EngineAuthError("操作查询参数无效", 400)
operation = await self._operation(
op_match[1], delegated, reconcile=query != b"reconcile=false")
await json_response(send, 200, self._public_operation(operation))
return
match = re.fullmatch(r"/internal/v1/hosts/([A-Za-z0-9][A-Za-z0-9_-]{0,63})/(.+)", path)
if match is None:
raise EngineAuthError("内部资源不存在", 404)
host_id, resource = match.groups()
if re.fullmatch(
r"user-isos(?:/import|/[0-9a-f]{32}|"
r"/import-jobs/[A-Za-z0-9_-]+)?", resource):
# User ISO ownership, quota, source bytes and deletion tombstones
# are engine-global. Keeping the historical host-prefixed alias
# reachable would let DELETE bypass replica GC and make the same
# catalog appear to have host-local semantics.
raise EngineAuthError("内部资源不存在", 404)
self._authorize_route(scope["method"], resource, delegated)
app = await self.registry.get(host_id)
if scope["method"] == "GET":
# Reads normally bypass the update coordination lock, but lazy
# cache GC mutates libvirt. Coordinate that maintenance work with
# online update and skip it while an update is already active.
async with self._update_lock():
if not await asyncio.to_thread(self.updater.active_job):
await self._cleanup_host_user_iso_tombstones(host_id)
await self._cleanup_host_cloud_template_tombstones(host_id)
else:
# Every host write reached this method while holding _update_lock
# and after the active-update gate in _dispatch().
await self._cleanup_host_user_iso_tombstones(host_id)
await self._cleanup_host_cloud_template_tombstones(host_id)
if resource.endswith("/console-ticket"):
vm_id = resource.split("/")[1]
await self._ticket(host_id, vm_id, delegated, send)
return
operation = None
create = scope["method"] == "POST" and resource == "vms"
mutation = scope["method"] == "DELETE" and resource.startswith("vms/") or (
scope["method"] == "POST" and bool(re.fullmatch(
rf"vms/{VM_ID}/(?:actions|reinstall|"
rf"(?:service|network)-(?:suspend|resume)|"
rf"rescue(?:/exit|/force-start)?)",
resource)))
if create or mutation:
if not OP_ID.fullmatch(delegated.get("operation_id", "")):
raise EngineAuthError("虚拟机操作缺少持久幂等编号", 403)
if create and not delegated.get("can_provision"):
raise EngineAuthError("虚拟机只能通过已确认套餐订单开通", 403)
operation, fresh = await self._reserve_operation(host_id, delegated, body,
resource=resource, method=scope["method"], target=raw_target(scope))
if operation.get("prevalidation_failed"):
await json_response(send, 422, self._public_operation(operation))
return
if not fresh:
if (operation["status"] in {"running", "pending", "uncertain"}
and self._service_target_method(operation.get("resource", ""))):
operation = await self._mutation_operation(operation)
if operation["status"] == "succeeded":
await json_response(send, 201 if create else 200,
{**operation["result"], **self._public_operation(operation)})
else:
await json_response(send, 202, self._public_operation(operation))
return
if create:
delegated["vm_uuid"] = operation["vm_uuid"]
self.active_operations.add(operation["operation_id"])
child_scope = {**scope, "path": "/api/v1/" + resource,
"raw_path": ("/api/v1/" + resource).encode(), "root_path": ""}
if operation is None:
# Register before saving so all persisted hosts are restored for
# unattended sampling after restart, even if the HTTP caller dies.
if resource == "host" and scope["method"] == "PUT":
await asyncio.to_thread(self.store.set, "engine_host", host_id, json.dumps({"host_id": host_id}))
await app(child_scope, receive, send)
return
messages = []
async def capture(message):
messages.append(message)
try:
target_method = self._service_target_method(operation.get("resource", ""))
# Every VM mutation shares the same lock, not just billing target
# state. A start/reset/delete that timed out at PHP may still be
# executing here when expiry enforcement arrives. Serializing the
# later policy call guarantees that its desired state wins. The
# reinstall/rescue endpoints register their per-VM pending guard
# before returning 202; child endpoints reject every other
# mutation until those background jobs remove that guard.
async with self._target_lock(host_id, operation["vm_uuid"]):
if target_method == "service_suspend":
await self._capture_service_was_running_hint(
host_id, operation, delegated)
await app(child_scope, receive, capture)
status = next(item["status"] for item in messages if item["type"] == "http.response.start")
payload = json.loads(b"".join(item.get("body", b"") for item in messages
if item["type"] == "http.response.body"))
operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else (
"failed" if not create and status < 500 else "uncertain")
if operation["status"] == "succeeded":
operation["result"] = payload
elif operation["status"] == "pending":
operation["result"] = payload
operation["job_id"] = payload["job_id"]
else:
operation["error"] = payload.get("detail") or payload.get("error") or "虚拟机开通失败"
if create:
await self._check_absent(operation)
operation["finished_at"] = int(time.time())
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
if operation["status"] in {"succeeded", "pending"}:
await json_response(send, status, {**payload, **self._public_operation(operation)})
else:
await json_response(send, status, {**payload, "operation_id": operation["operation_id"],
"status": operation["status"], "safe_to_refund": operation["safe_to_refund"]})
except BaseException:
operation["status"] = "uncertain"
operation["error"] = "开通结果不确定,请查询操作状态,不要重复创建"
with suppress(Exception):
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
raise
finally:
self.active_operations.discard(operation["operation_id"])
@staticmethod
def _cloud_upload_metadata(scope) -> tuple[str, str, str]:
query = scope.get("query_string", b"")
if (not isinstance(query, bytes) or len(query) > 2048
or re.search(rb"%(?![0-9A-Fa-f]{2})", query) is not None):
raise TemplateError("云镜像上传参数无效", 422)
try:
pairs = parse_qsl(
query.decode("ascii"),
keep_blank_values=True,
strict_parsing=True,
max_num_fields=3,
errors="strict",
)
except (UnicodeDecodeError, ValueError) as exc:
raise TemplateError("云镜像上传参数无效", 422) from exc
if (len(pairs) != 3
or {key for key, _value in pairs}
!= {"name", "os_family", "os_version"}):
raise TemplateError("云镜像上传参数无效", 422)
values = dict(pairs)
name = validate_template_name(values["name"])
os_family, os_version = validate_template_metadata(
values["os_family"], values["os_version"])
return name, os_family, os_version
@staticmethod
def _single_upload_header(scope, name: bytes) -> str | None:
values = [value.decode("latin-1")
for key, value in scope.get("headers", [])
if key.lower() == name]
if len(values) > 1:
raise TemplateError("云镜像上传请求头重复", 400)
return values[0] if values else None
async def _upload_cloud_template(self, scope, receive, send) -> None:
runtime = self.registry.cloud_templates
name, os_family, os_version = self._cloud_upload_metadata(scope)
content_type = self._single_upload_header(scope, b"content-type")
if content_type is None or content_type.lower() != "application/octet-stream":
raise TemplateError(
"云镜像上传必须使用 application/octet-stream", 415)
content_encoding = self._single_upload_header(scope, b"content-encoding")
if content_encoding is not None and content_encoding.lower() != "identity":
raise TemplateError("云镜像上传内容不能压缩", 415)
declared_raw = self._single_upload_header(scope, b"content-length")
if declared_raw is not None and re.fullmatch(r"[0-9]+", declared_raw) is None:
raise TemplateError("云镜像上传长度无效", 400)
declared = int(declared_raw) if declared_raw is not None else None
if declared is not None and declared < MIN_DOWNLOAD_BYTES:
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
if declared is not None and declared > MAX_IMAGE_BYTES:
raise TemplateError("云镜像超过 4 GiB 限制", 413)
async with self._update_lock():
if await asyncio.to_thread(self.updater.active_job):
raise EngineAuthError(
"面板正在更新,暂时不能执行虚拟化写操作", 409)
reservation = await asyncio.to_thread(
runtime.reserve_upload, name, os_family, os_version)
reservation_id = reservation["reservation_id"]
maximum = reservation["max_bytes"]
if declared is not None and declared > maximum:
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
raise TemplateError("面板云镜像持久目录空间不足", 507)
image_path: Path | None = None
handed_off = False
writer: _CloudUploadWriter | None = None
try:
writer = await asyncio.to_thread(
_CloudUploadWriter, reservation["directory"])
image_path = writer.path
checksum = hashlib.sha256()
size = 0
deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS
try:
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TemplateError("云镜像上传超时", 408)
try:
message = await asyncio.wait_for(
receive(),
timeout=min(CLOUD_UPLOAD_IDLE_SECONDS, remaining),
)
except TimeoutError as exc:
raise TemplateError("云镜像上传超时", 408) from exc
if message["type"] == "http.disconnect":
raise TemplateError("云镜像上传已中断", 400)
if message["type"] != "http.request":
raise TemplateError("云镜像上传请求无效", 400)
chunk = message.get("body", b"")
size += len(chunk)
if declared is not None and size > declared:
raise TemplateError("云镜像上传内容超过声明长度", 400)
if size > maximum:
if maximum < MAX_IMAGE_BYTES:
raise TemplateError(
"面板云镜像持久目录空间不足", 507)
raise TemplateError("云镜像超过 4 GiB 限制", 413)
view = memoryview(chunk)
for offset in range(0, len(view), CLOUD_UPLOAD_WRITE_BYTES):
if time.monotonic() >= deadline:
raise TemplateError("云镜像上传超时", 408)
part = view[offset:offset + CLOUD_UPLOAD_WRITE_BYTES]
await asyncio.to_thread(writer.write, part)
checksum.update(part)
if not message.get("more_body", False):
break
await asyncio.to_thread(writer.sync_and_close)
except OSError as exc:
if exc.errno in {errno.ENOSPC, errno.EDQUOT}:
raise TemplateError(
"面板云镜像持久目录空间不足", 507) from exc
raise
if declared is not None and size != declared:
raise TemplateError("云镜像上传内容不完整", 400)
if size < MIN_DOWNLOAD_BYTES:
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
validation = scope.get("engine_upload_validation", {})
expected_digest = scope.get("engine_upload_digest", "")
digest = checksum.hexdigest()
if (not validation.get("complete")
or not validation.get("valid")
or not hmac.compare_digest(digest, expected_digest)):
raise EngineAuthError("云镜像摘要与签名不匹配", 401)
virtual_size = await asyncio.to_thread(
verify_qcow2_image, image_path)
# Recheck the update state at the commit boundary. An upload
# admitted before a service transition must not enqueue work
# against code that is being replaced.
async with self._update_lock():
if await asyncio.to_thread(self.updater.active_job):
raise EngineAuthError(
"面板正在更新,暂时不能执行虚拟化写操作", 409)
result = await asyncio.to_thread(
runtime.queue_upload,
reservation_id,
image_path,
digest,
virtual_size,
)
handed_off = True
await json_response(send, 202, result)
finally:
if not handed_off:
if writer is not None:
with suppress(OSError):
await asyncio.to_thread(writer.close)
if image_path is not None:
with suppress(OSError):
await asyncio.to_thread(image_path.unlink, missing_ok=True)
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
async def _cloud_template_dispatch(
self, method: str, resource: str, body: bytes | None, send,
*, scope=None, receive=None) -> None:
runtime = self.registry.cloud_templates
try:
if method == "GET" and resource == "vm-templates":
records = await asyncio.to_thread(runtime.list)
# A normal catalog read also repairs the narrow crash/lost-ack
# window between source deletion and its final tombstone flag.
# Once repaired, immediately resume cache GC on every loaded
# host instead of requiring an engine restart.
for host_id in tuple(self.registry.components):
self._background(
self._cleanup_host_cloud_template_tombstones(host_id))
await json_response(send, 200, {"templates": records})
return
if method == "POST" and resource == "vm-templates/import":
from .app import ImportTemplateBody
from pydantic import ValidationError
try:
value = ImportTemplateBody.model_validate_json(body or b"")
except ValidationError as exc:
raise TemplateError("云镜像导入参数无效", 422) from exc
result = await asyncio.to_thread(
runtime.queue_import, value.url, value.name, value.sha256)
await json_response(send, 202, result)
return
if method == "POST" and resource == "vm-templates/upload":
if scope is None or receive is None:
raise EngineAuthError("云镜像上传请求无效", 400)
await self._upload_cloud_template(scope, receive, send)
return
job_match = re.fullmatch(
r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource)
if method == "GET" and job_match is not None:
result = await asyncio.to_thread(
runtime.get_job, job_match[1])
await json_response(send, 200, result)
return
template_match = re.fullmatch(
r"vm-templates/([A-Za-z0-9_-]+)", resource)
if method == "PATCH" and template_match is not None:
from .app import RenameTemplateBody
from pydantic import ValidationError
try:
value = RenameTemplateBody.model_validate_json(body or b"")
except ValidationError as exc:
raise TemplateError("云镜像名称无效", 422) from exc
record = await asyncio.to_thread(
runtime.rename, template_match[1], value.name)
await json_response(send, 200, {"template": record})
return
if method == "DELETE" and template_match is not None:
from .app import DeleteTemplateBody
from pydantic import ValidationError
try:
value = DeleteTemplateBody.model_validate_json(body or b"")
except ValidationError as exc:
raise TemplateError("云镜像删除参数无效", 422) from exc
template_id = template_match[1]
tombstone_key, tombstone = await asyncio.to_thread(
self._delete_cloud_template_source,
template_id, value.created_at)
for host_id in tombstone["pending_hosts"]:
self._background(
self._cleanup_cloud_template_tombstone_on_host(
tombstone_key, tombstone, host_id))
await json_response(send, 200, {
"deleted": True,
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
})
return
raise EngineAuthError("内部资源不存在", 404)
except TemplateError as exc:
await json_response(send, exc.status, {"error": str(exc)})
async def _user_iso_dispatch(self, method: str, resource: str,
body: bytes | None, delegated: dict, send) -> None:
runtime = self.registry.user_isos
user_id = delegated["user_id"]
try:
if method == "GET" and resource == "user-isos":
result = await asyncio.to_thread(runtime.list_for, user_id)
await json_response(send, 200, result)
return
if method == "POST" and resource == "user-isos/import":
from .app import ImportUserISOBody
from pydantic import ValidationError
try:
value = ImportUserISOBody.model_validate_json(body or b"")
except ValidationError as exc:
raise TemplateError("ISO 导入参数无效", 422) from exc
result = await asyncio.to_thread(
runtime.queue_import, user_id, value.url, value.name,
value.sha256)
await json_response(send, 202, result)
return
job_match = re.fullmatch(
r"user-isos/import-jobs/([A-Za-z0-9_-]+)", resource)
if method == "GET" and job_match is not None:
result = await asyncio.to_thread(
runtime.get_job_for, user_id, job_match[1])
await json_response(send, 200, result)
return
iso_match = re.fullmatch(r"user-isos/([0-9a-f]{32})", resource)
if iso_match is not None and method == "GET":
record = await asyncio.to_thread(
runtime.get_for, user_id, iso_match[1])
await json_response(
send, 200, {"iso": runtime.public_record(record)})
return
if iso_match is not None and method == "DELETE":
iso_id = iso_match[1]
tombstone_key, tombstone = await asyncio.to_thread(
self._delete_user_iso_source, user_id, iso_id)
for host_id in tombstone["pending_hosts"]:
self._background(self._cleanup_user_iso_tombstone_on_host(
tombstone_key, tombstone, host_id))
await json_response(send, 200, {
"deleted": True,
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
})
return
raise EngineAuthError("内部资源不存在", 404)
except TemplateError as exc:
await json_response(send, exc.status, {"error": str(exc)})
def _delete_cloud_template_source(
self, template_id: str,
expected_created_at: int) -> tuple[str, dict]:
"""Commit global source deletion and durable per-host cache GC."""
runtime = self.registry.cloud_templates
with runtime.lock:
raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id)
if raw is not None:
tombstone = self._decode_cloud_template_tombstone(
template_id, raw)
if tombstone["template_created_at"] != expected_created_at:
raise TemplateError(
"云镜像版本已变化,请刷新后重试", 409)
if tombstone["source_deleted"]:
return template_id, tombstone
try:
record = runtime.get(template_id)
except TemplateError as exc:
if raw is None or exc.status != 404:
raise
tombstone["source_deleted"] = True
self.store.set(
CACHE_TOMBSTONE_KIND, template_id,
json.dumps(tombstone, separators=(",", ":")),
)
return template_id, tombstone
source_volume_name = record.get("volume_name")
if source_volume_name != template_volume_name(template_id):
raise TemplateError("云镜像目录记录损坏,无法安全删除", 500)
if record.get("created_at") != expected_created_at:
raise TemplateError("云镜像版本已变化,请刷新后重试", 409)
host_ids = sorted(
set(self.store.list("engine_host"))
| set(self.registry.components)
)
tombstone = {
"deletion_id": uuid.uuid4().hex,
"template_id": template_id,
"template_created_at": expected_created_at,
"source_volume_name": source_volume_name,
"pending_hosts": host_ids,
"source_deleted": False,
"created_at": int(time.time()),
}
encoded = json.dumps(tombstone, separators=(",", ":"))
self.store.set(CACHE_TOMBSTONE_KIND, template_id, encoded)
try:
# Re-enters runtime.lock, which is intentionally an RLock.
runtime.delete(template_id)
except Exception:
# A database DELETE acknowledgement can be lost before or
# after commit. The catalog alone is not sufficient proof of
# rollback because source unlink happens first. Reconcile both
# durable halves: only an intact catalog+source pair cancels
# this marker; a missing source completes retirement and keeps
# host GC durable.
if self.store.get(
CACHE_TOMBSTONE_KIND, template_id) == encoded:
reconciled = runtime._reconcile_cache_tombstone(
template_id, tombstone.copy())
if (reconciled is not None
and reconciled["source_deleted"]):
return template_id, reconciled
raise
tombstone["source_deleted"] = True
self.store.set(
CACHE_TOMBSTONE_KIND, template_id,
json.dumps(tombstone, separators=(",", ":")),
)
return template_id, tombstone
@staticmethod
def _decode_cloud_template_tombstone(key: str, raw: str) -> dict:
try:
value = json.loads(raw)
deletion_id = value["deletion_id"]
template_id = value["template_id"]
template_created_at = value["template_created_at"]
pending = value["pending_hosts"]
except (KeyError, TypeError, ValueError) as exc:
raise ValueError("cloud template tombstone is corrupt") from exc
try:
expected_volume_name = template_volume_name(template_id)
except TemplateError as exc:
raise ValueError("cloud template tombstone is corrupt") from exc
if (not isinstance(deletion_id, str)
or not re.fullmatch(r"[0-9a-f]{32}", deletion_id)
or key != template_id
or isinstance(template_created_at, bool)
or not isinstance(template_created_at, int)
or template_created_at < 1
or value.get("source_volume_name") != expected_volume_name
or not isinstance(pending, list)
or any(not isinstance(host_id, str)
or not HOST_ID.fullmatch(host_id)
for host_id in pending)
or len(set(pending)) != len(pending)
or not isinstance(value.get("source_deleted"), bool)):
raise ValueError("cloud template tombstone is corrupt")
return value
async def _cleanup_cloud_template_tombstone_on_host(
self, key: str, tombstone: dict, host_id: str) -> None:
if (not tombstone.get("source_deleted")
or host_id not in tombstone["pending_hosts"]):
return
try:
await self.registry.get(host_id)
components = self.registry.components[host_id]
profile = await asyncio.to_thread(components.manager.current)
if profile is None or profile.get("key_locked"):
return
service = self._service(profile)
await asyncio.to_thread(
self._cleanup_cloud_template_tombstone_on_host_sync,
key, tombstone["deletion_id"], host_id, service, components)
except Exception as exc:
LOG.info(
"Cloud template cache cleanup deferred: host=%s type=%s",
host_id, type(exc).__name__)
def _cleanup_cloud_template_tombstone_on_host_sync(
self, key: str, deletion_id: str, host_id: str,
service, components: EngineComponents) -> None:
"""Revalidate the deletion generation at the libvirt commit point.
Duplicate cleanup tasks can outlive the tombstone snapshot that
scheduled them. Holding the shared runtime lock across the final
re-read, libvirt deletion and marker update prevents an obsolete task
from deleting a replica created by a same-content reimport.
"""
runtime = self.registry.cloud_templates
with runtime.lock:
raw = self.store.get(CACHE_TOMBSTONE_KIND, key)
if raw is None:
return
value = self._decode_cloud_template_tombstone(key, raw)
if (value["deletion_id"] != deletion_id
or not value["source_deleted"]
or host_id not in value["pending_hosts"]):
return
try:
service.delete_cloud_template_replicas(
value["source_volume_name"])
except Exception as exc:
from .libvirt_service import VMError
# Only an accessible pool with no matching volumes is an
# idempotent success. A missing/misconfigured storage pool is
# also reported as 404 by libvirt_service and must keep this
# host pending so an old replica cannot escape reclamation.
if (not isinstance(exc, VMError)
or exc.status != 404
or str(exc) != "云镜像宿主机缓存不存在"):
raise
components.store.delete(
"cloud_template_replica", value["template_id"])
value["pending_hosts"] = [
candidate for candidate in value["pending_hosts"]
if candidate != host_id
]
# Retain the terminal record so a response lost after source
# deletion remains retryable until an intentional reimport.
self.store.set(
CACHE_TOMBSTONE_KIND, key,
json.dumps(value, separators=(",", ":")),
)
async def _cleanup_host_cloud_template_tombstones(
self, host_id: str) -> None:
rows = await asyncio.to_thread(
self.store.list, CACHE_TOMBSTONE_KIND)
for key, raw in rows.items():
try:
tombstone = self._decode_cloud_template_tombstone(key, raw)
except ValueError:
LOG.error(
"Corrupt cloud template cache cleanup record: key=%s", key)
continue
if host_id in tombstone["pending_hosts"]:
await self._cleanup_cloud_template_tombstone_on_host(
key, tombstone, host_id)
def _delete_user_iso_source(
self, user_id: int, iso_id: str) -> tuple[str, dict]:
"""Commit one source deletion and its remote-GC tombstone atomically.
The shared runtime lock serializes the whole transition. In
particular, a second DELETE can only observe the first request's
committed tombstone; it must never remove that durable cleanup record
after the catalog has disappeared.
"""
runtime = self.registry.user_isos
key = f"{user_id}:{iso_id}"
with runtime.lock:
raw = self.store.get("user_iso_tombstone", key)
if raw is not None:
tombstone = self._decode_user_iso_tombstone(key, raw)
if tombstone["source_deleted"]:
return key, tombstone
try:
record = runtime.get_for(user_id, iso_id)
except TemplateError as exc:
if raw is None or exc.status != 404:
raise
# The source/catalog removal committed but persisting the
# final flag failed. Preserve and advance the existing GC
# record instead of erasing it on a repeated DELETE.
tombstone["source_deleted"] = True
self.store.set("user_iso_tombstone", key, json.dumps(
tombstone, separators=(",", ":")))
if not tombstone["pending_hosts"]:
self.store.delete("user_iso_tombstone", key)
return key, tombstone
host_ids = self._user_iso_replica_hosts(user_id, record)
tombstone = {
"user_id": user_id,
"iso_id": iso_id,
"volume_name": record["volume_name"],
"pending_hosts": host_ids,
"source_deleted": False,
"created_at": int(time.time()),
}
encoded = json.dumps(tombstone, separators=(",", ":"))
self.store.set("user_iso_tombstone", key, encoded)
try:
# Re-enters runtime.lock; it is intentionally an RLock.
runtime.delete_for(user_id, iso_id)
except Exception:
# Only erase the exact provisional record written above. A
# later committed tombstone must never be removed by a failed
# or repeated request.
if self.store.get("user_iso_tombstone", key) == encoded:
self.store.delete("user_iso_tombstone", key)
raise
tombstone["source_deleted"] = True
self.store.set("user_iso_tombstone", key, json.dumps(
tombstone, separators=(",", ":")))
if not host_ids:
self.store.delete("user_iso_tombstone", key)
return key, tombstone
def _user_iso_replica_hosts(self, user_id: int,
record: dict) -> list[str]:
replica_key = f"{user_id}:{record['id']}"
result = []
host_ids = set(self.store.list("engine_host")) | set(self.registry.components)
for host_id in host_ids:
raw = HostStore(self.store, host_id).get(
"user_iso_replica", replica_key)
if raw is None:
continue
# The exact owner/ISO marker key is already proof that this host
# may contain the deterministic volume. Reuse requires a strict
# ready marker, but deletion must fail safe for old or partially
# damaged marker schemas as well.
result.append(host_id)
return sorted(result)
@staticmethod
def _decode_user_iso_tombstone(key: str, raw: str) -> dict:
from .user_isos import user_iso_volume_name
try:
value = json.loads(raw)
user_id = value["user_id"]
iso_id = value["iso_id"]
pending = value["pending_hosts"]
except (KeyError, TypeError, ValueError) as exc:
raise ValueError("user ISO tombstone is corrupt") from exc
if (isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1
or key != f"{user_id}:{iso_id}"
or value.get("volume_name") != user_iso_volume_name(iso_id)
or not isinstance(pending, list)
or any(not isinstance(host_id, str) or not HOST_ID.fullmatch(host_id)
for host_id in pending)
or len(set(pending)) != len(pending)
or not isinstance(value.get("source_deleted"), bool)):
raise ValueError("user ISO tombstone is corrupt")
return value
async def _cleanup_user_iso_tombstone_on_host(
self, key: str, tombstone: dict, host_id: str) -> None:
if not tombstone.get("source_deleted") or host_id not in tombstone["pending_hosts"]:
return
try:
await self.registry.get(host_id)
components = self.registry.components[host_id]
profile = await asyncio.to_thread(components.manager.current)
if profile is None or profile.get("key_locked"):
return
service = self._service(profile)
try:
await asyncio.to_thread(
service.delete_user_iso, tombstone["user_id"],
tombstone["volume_name"])
except Exception as exc:
from .libvirt_service import VMError
if not isinstance(exc, VMError) or exc.status != 404:
LOG.info(
"User ISO cache cleanup remains pending: host=%s type=%s",
host_id, type(exc).__name__)
return
await asyncio.to_thread(
components.store.delete, "user_iso_replica",
f"{tombstone['user_id']}:{tombstone['iso_id']}")
await asyncio.to_thread(
self._complete_user_iso_tombstone_host, key, host_id)
except Exception as exc:
LOG.info("User ISO cache cleanup deferred: host=%s type=%s",
host_id, type(exc).__name__)
def _complete_user_iso_tombstone_host(self, key: str, host_id: str) -> None:
runtime = self.registry.user_isos
with runtime.lock:
raw = self.store.get("user_iso_tombstone", key)
if raw is None:
return
value = self._decode_user_iso_tombstone(key, raw)
value["pending_hosts"] = [
candidate for candidate in value["pending_hosts"]
if candidate != host_id
]
if value["pending_hosts"]:
self.store.set("user_iso_tombstone", key, json.dumps(
value, separators=(",", ":")))
else:
self.store.delete("user_iso_tombstone", key)
async def _cleanup_host_user_iso_tombstones(self, host_id: str) -> None:
rows = await asyncio.to_thread(self.store.list, "user_iso_tombstone")
for key, raw in rows.items():
try:
tombstone = self._decode_user_iso_tombstone(key, raw)
except ValueError:
LOG.error("Corrupt user ISO cache cleanup record: key=%s", key)
continue
if host_id in tombstone["pending_hosts"]:
await self._cleanup_user_iso_tombstone_on_host(
key, tombstone, host_id)
@staticmethod
def _update_object(body: bytes | None, *, empty: bool = False) -> dict:
if (body is None or body == b"") and empty:
return {}
try:
value = json.loads(body or b"")
except (TypeError, ValueError, UnicodeError) as exc:
raise UpdateError("在线更新参数必须是 JSON 对象", 422) from exc
if not isinstance(value, dict):
raise UpdateError("在线更新参数必须是 JSON 对象", 422)
return value
@staticmethod
def _public_update_job(job: dict) -> dict:
allowed = {
"job_id", "status", "phase", "created_at", "started_at", "finished_at",
"updated", "old_commit", "new_commit", "rollback_ref", "error",
}
return {key: value for key, value in job.items() if key in allowed}
def _long_running_host_job(self) -> bool:
cloud_templates = getattr(self.registry, "cloud_templates", None)
if (self.maintenance_tasks
or self.registry.user_isos.has_active_jobs()
or (cloud_templates is not None
and cloud_templates.has_active_jobs())):
return True
for components in self.registry.components.values():
kinds = ["rescue_import_job", "reinstall_job", "rescue_job"]
# A child app without the shared runtime can still run the legacy
# host-scoped importer. Production children use the global runtime;
# stale pre-upgrade import_job rows must not block updates forever.
if components.cloud_templates is None:
kinds.append("import_job")
for kind in kinds:
for raw in components.store.list(kind).values():
try:
status = json.loads(raw).get("status")
except (AttributeError, TypeError, ValueError):
return True
if status in {"queued", "running", "pending"}:
return True
return False
async def _update_dispatch(self, method: str, path: str, body: bytes | None,
delegated: dict, send) -> None:
if not (delegated.get("is_admin") and delegated.get("is_super_admin")
and delegated.get("can_manage")):
raise EngineAuthError("仅超级管理员可以管理在线更新", 403)
if path == "/internal/v1/update/settings" and method == "GET":
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
return
if path == "/internal/v1/update/settings" and method == "PUT":
value = self._update_object(body)
if set(value) - {"access_token", "clear_token"}:
raise UpdateError("在线更新设置包含未知字段", 422)
clear = value.get("clear_token", False)
token = value.get("access_token")
if not isinstance(clear, bool) or (token is not None and not isinstance(token, str)):
raise UpdateError("在线更新设置格式无效", 422)
if clear and token not in {None, ""}:
raise UpdateError("不能同时保存和删除访问令牌", 422)
if clear:
await asyncio.to_thread(self.updater.clear_token)
elif token not in {None, ""}:
await asyncio.to_thread(self.updater.set_token, token)
elif "access_token" not in value:
raise UpdateError("没有需要保存的在线更新设置", 422)
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
return
if path == "/internal/v1/update/check" and method == "POST":
value = self._update_object(body, empty=True)
if value:
raise UpdateError("检查更新不接受额外参数", 422)
if await asyncio.to_thread(self.updater.active_job):
raise UpdateError("已有更新任务正在执行", 409)
await json_response(send, 200, await asyncio.to_thread(self.updater.check))
return
if path == "/internal/v1/update/apply" and method == "POST":
value = self._update_object(body)
if set(value) != {"expected_current_commit", "expected_remote_commit"}:
raise UpdateError("请先检查更新并提交已确认的版本", 422)
if not all(isinstance(value[key], str) for key in value):
raise UpdateError("更新版本标识无效", 422)
async with self._update_lock():
if self.active_operations or await asyncio.to_thread(self._long_running_host_job):
raise UpdateError("仍有虚拟机任务正在执行,请等待任务完成后再更新", 409)
job = await asyncio.to_thread(
self.updater.create_job,
expected_current_commit=value["expected_current_commit"],
expected_remote_commit=value["expected_remote_commit"],
)
try:
await asyncio.to_thread(self.update_trigger)
job = await self._wait_update_worker_started(job["job_id"])
except UpdateServiceUncertain:
raise
except UpdateError as exc:
await asyncio.to_thread(
self.updater.fail_job, job["job_id"],
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
)
raise exc
except Exception as exc:
await asyncio.to_thread(
self.updater.fail_job, job["job_id"],
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
)
raise UpdateError(
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
) from exc
await json_response(send, 202, self._public_update_job(job))
return
job_match = re.fullmatch(
r"/internal/v1/update/jobs/([A-Za-z0-9_-]{16,64})", path)
if method == "GET" and job_match:
job = await asyncio.to_thread(self.updater.get_job, job_match[1])
if job is None:
raise UpdateError("更新任务不存在", 404)
await json_response(send, 200, self._public_update_job(job))
return
raise EngineAuthError("内部资源不存在", 404)
async def _wait_update_worker_started(self, job_id: str) -> dict:
"""Confirm the one-shot claimed the durable job, not merely its queue."""
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
job = await asyncio.to_thread(self.updater.get_job, job_id)
if job is None:
raise UpdateError("独立更新服务未生成任务状态", 503)
if job.get("status") in {"running", "succeeded"}:
return job
if job.get("status") == "failed":
raise UpdateError("独立更新服务启动后失败,请查看更新服务日志", 503)
if job.get("status") == "blocked":
raise UpdateServiceUncertain(
"更新任务中断且维护锁仍在生效,请通过 SSH 检查更新服务和回滚引用",
503,
)
await asyncio.sleep(0.1)
# Stop any very-late process before releasing the maintenance gate and
# marking the job failed. This prevents an unclaimed worker from
# changing code after ordinary VM mutations have resumed.
await asyncio.to_thread(self._stop_update_service)
raise UpdateError("独立更新服务未在限定时间内启动,请查看 systemd 日志", 503)
@staticmethod
def _start_update_service() -> None:
try:
result = subprocess.run(
["/usr/bin/systemctl", "--no-block", "start", "virt-cloud-update.service"],
cwd="/",
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=15,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise UpdateError(
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
) from exc
if result.returncode != 0:
raise UpdateError(
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
@staticmethod
def _update_service_active() -> bool | None:
try:
result = subprocess.run(
["/usr/bin/systemctl", "show", "--property=ActiveState", "--value",
"virt-cloud-update.service"],
cwd="/",
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=5,
check=False,
)
except (OSError, subprocess.TimeoutExpired):
return None
if result.returncode != 0:
return None
state = result.stdout.strip()
if state in {"active", "activating", "deactivating", "reloading"}:
return True
if state in {"inactive", "failed"}:
return False
return None
@staticmethod
def _stop_update_service() -> None:
try:
result = subprocess.run(
["/usr/bin/systemctl", "stop", "virt-cloud-update.service"],
cwd="/",
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
capture_output=True,
text=True,
timeout=30,
check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise UpdateServiceUncertain(
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503) from exc
if result.returncode != 0:
raise UpdateServiceUncertain(
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503)
async def _reserve_operation(self, host_id, delegated, body, *, resource="vms", method="POST", target=""):
op_id = delegated["operation_id"]
# Even rejected signed provisioning requests have a durable outcome.
# A worker that loses this HTTP response can release its frozen balance
# only by querying this same operation, without replaying the request.
from .app import CreateCloudBody, CreateISOBody
from pydantic import ValidationError
create = resource == "vms"
value = {}
error = None
try:
value = json.loads(body) if body else {}
if not isinstance(value, dict):
raise ValueError("body must be an object")
if create:
(CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value)
if value.get("guest_ssh_key_id") is not None or value.get("ssh_key_id") is not None:
error = "公钥必须由 PHP 校验用户归属后传递,不能使用旧公钥 ID"
except (ValueError, AttributeError, TypeError, ValidationError):
value = {}
error = "虚拟机参数校验失败"
record = {"operation_id": op_id, "host_id": host_id, "user_id": delegated["user_id"],
"request_hash": hmac.new(self.secret, method.encode() + b"\n" + target.encode() + b"\n" + (body or b""), hashlib.sha256).hexdigest(),
"vm_uuid": str(uuid.uuid4()) if create else str(uuid.UUID(resource.split("/")[1])),
"vm_name": value.get("name"), "kind": "create" if create else "mutation",
"actor_can_manage": bool(delegated.get("can_manage")),
"actor_is_admin": bool(delegated.get("is_admin")),
"resource": resource, "method": method, "status": "running", "result": None,
"safe_to_refund": False, "created_at": int(time.time())}
if create:
# New PHP-backed creates always receive the reserved UUID below.
# Keep this durable marker so cleanup reconciliation checks the
# UUID-derived resource candidates, while old in-flight records
# continue to use their historical display-name disk convention.
record["resource_naming"] = "uuid-v1"
if resource.endswith("/service-resume"):
# Store only the allowlisted target-state option. It is needed to
# replay the exact same convergence request after a lost response;
# never persist arbitrary request bodies in the operation table.
record["target_options"] = {
"start_if_suspended": value.get("start_if_suspended") is True,
}
if error:
# No service, task, VM or volume mutation has been invoked.
record.update(status="failed", prevalidation_failed=True,
error=error, safe_to_refund=create, finished_at=int(time.time()))
fresh = await asyncio.to_thread(self.store.insert, "operation", op_id, json.dumps(record))
if fresh:
return record, True
previous = await self._operation(op_id, delegated, reconcile=False)
if previous["host_id"] != host_id or previous["user_id"] != delegated["user_id"] or previous["request_hash"] != record["request_hash"]:
raise EngineAuthError("相同开通操作编号的请求内容不一致", 409)
return previous, False
async def _operation(self, op_id, delegated, *, reconcile=True):
raw = await asyncio.to_thread(self.store.get, "operation", op_id)
if raw is None:
raise EngineAuthError("开通操作不存在", 404)
record = json.loads(raw)
if record["user_id"] != delegated["user_id"] and not delegated["is_admin"]:
raise EngineAuthError("开通操作不存在", 404)
# Never automatically create again. Recover a known UUID if libvirt
# committed while the response or process was lost.
if reconcile and record.get("kind") == "mutation" and record["status"] in {"running", "pending", "uncertain"}:
return await self._mutation_operation(record)
if (reconcile and record["status"] in {"running", "uncertain"}
and op_id not in self.active_operations
and (record["status"] == "uncertain" or int(time.time()) - record.get("created_at", 0) >= 120)):
await self.registry.get(record["host_id"])
components = self.registry.components[record["host_id"]]
profile = components.manager.current()
if profile and not profile.get("key_locked"):
try:
vm = await asyncio.to_thread(self._service(profile).get_vm, record["vm_uuid"])
if vm.get("state") != "running" or not vm.get("managed"):
return record
from .metrics import host_scope
await asyncio.to_thread(components.state.users.assign_vm,
host_scope(profile), record["vm_uuid"], record["user_id"])
record.update(status="succeeded", result=vm, safe_to_refund=False)
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
except Exception:
# A missing VM can be a transient connection failure or an
# operation still in progress. It is not permission to retry.
await self._check_absent(record)
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
return record
async def _mutation_operation(self, record):
await self.registry.get(record["host_id"])
components = self.registry.components[record["host_id"]]
target_method = self._service_target_method(record.get("resource", ""))
if target_method is not None:
if record["operation_id"] in self.active_operations:
return record
if (target_method == "service_suspend"
and not isinstance(record.get("service_was_running_hint"), bool)):
# Suspending without a durable pre-stop state can permanently
# lose whether a later paid renewal is allowed to restart the
# guest. Fail closed instead of replaying a destructive target
# after the database recovers.
record.update(
status="failed",
error="停机前运行状态未安全保存,已取消服务暂停",
finished_at=int(time.time()),
)
await asyncio.to_thread(
self.store.set, "operation", record["operation_id"],
json.dumps(record))
return record
profile = components.manager.current()
if profile is None or profile.get("key_locked"):
return record
if (not record.get("actor_can_manage")
or (not record.get("actor_is_admin")
and not components.state.users.owns_vm(
self._host_scope(profile), record["vm_uuid"], record["user_id"]))):
record.update(status="failed", error="虚拟机不存在", finished_at=int(time.time()))
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
json.dumps(record))
return record
self.active_operations.add(record["operation_id"])
try:
async with self._target_lock(record["host_id"], record["vm_uuid"]):
options = record.get("target_options", {}) \
if target_method == "service_resume" else {}
if not isinstance(options, dict) or any(
key != "start_if_suspended" for key in options):
options = {}
result = await asyncio.to_thread(
getattr(self._service(profile), target_method), record["vm_uuid"],
**options)
if (target_method == "service_suspend"
and isinstance(record.get("service_was_running_hint"), bool)):
result["service_was_running"] = (
bool(result.get("service_was_running"))
or record["service_was_running_hint"])
except Exception as exc:
from .libvirt_service import VMError
if isinstance(exc, VMError) and exc.status < 500:
record.update(status="failed", error=str(exc), finished_at=int(time.time()))
else:
record.update(status="uncertain",
error="服务状态尚未确认,引擎将安全重试")
else:
record.update(status="succeeded", result=result, error=None,
finished_at=int(time.time()))
finally:
self.active_operations.discard(record["operation_id"])
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
json.dumps(record))
return record
kind = "reinstall_job" if record["resource"].endswith("/reinstall") else "rescue_job" if "/rescue" in record["resource"] else None
if kind is None:
# A reset/shutdown/delete cannot be safely replayed merely because
# its response disappeared. Once no executor can still own the
# operation, terminate it visibly instead of allowing a permanent
# running/uncertain row to defeat expiry enforcement.
return await self._finish_orphan_mutation(record)
scoped = components.store
job = None
job_id = record.get("job_id")
if job_id:
raw = await asyncio.to_thread(scoped.get, kind, job_id)
job = json.loads(raw) if raw else None
else:
rows = await asyncio.to_thread(scoped.list, kind)
for candidate_id, raw in rows.items():
candidate = json.loads(raw)
if candidate.get("operation_id") == record["operation_id"]:
job_id, job = candidate_id, candidate
break
if job is None:
return await self._finish_orphan_mutation(record)
if job.get("owner") != f"php:{record['user_id']}" or job.get("vm_id") != record["vm_uuid"]:
return record
record["job_id"] = job_id
record["result"] = {key: value for key, value in job.items() if key not in {"scope", "owner", "operation_id"}}
record["result"]["job_id"] = job_id
status = job.get("status")
record["status"] = "succeeded" if status in {"completed", "succeeded"} else "running" if status in {"queued", "running"} else "failed"
if "面板重启" in job.get("phase", "") or "面板在任务执行期间重启" in job.get("error", ""):
# The asynchronous job registry explicitly confirms that this job
# is no longer running. Replaying a reinstall/rescue mutation is
# unsafe, but leaving it uncertain forever would also grant free
# service by permanently blocking expiry enforcement.
record["status"] = "failed"
record["error"] = job.get("error") or "引擎重启导致任务中断,请核查虚拟机状态后重试"
if job.get("error"):
record["error"] = job["error"]
await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record))
return record
async def _finish_orphan_mutation(self, record):
if (record["operation_id"] in self.active_operations
or int(time.time()) - int(record.get("created_at", 0)) < 120):
return record
record.update(
status="failed",
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
finished_at=int(time.time()),
)
await asyncio.to_thread(
self.store.set, "operation", record["operation_id"], json.dumps(record))
return record
@staticmethod
def _service_target_method(resource: str) -> str | None:
match = re.fullmatch(
rf"vms/{VM_ID}/((?:service|network)-(?:suspend|resume))", resource)
if match is None:
return None
return {
"service-suspend": "service_suspend",
"service-resume": "service_resume",
"network-suspend": "network_suspend",
"network-resume": "network_resume",
}[match[1]]
@staticmethod
def _host_scope(profile):
from .metrics import host_scope
return host_scope(profile)
async def _capture_service_was_running_hint(self, host_id, operation, delegated):
# Run this inside the same per-VM lock as service_suspend. If the
# engine crashes after stopping QEMU but before saving the response, a
# replay otherwise sees only a shutoff guest and cannot know whether a
# later renewal is allowed to start it automatically.
components = self.registry.components[host_id]
profile = components.manager.current()
owns = (profile is not None and not profile.get("key_locked")
and (delegated.get("is_admin") or components.state.users.owns_vm(
self._host_scope(profile), operation["vm_uuid"], delegated["user_id"])))
if not owns:
return
before = await asyncio.to_thread(
self._service(profile).get_vm, operation["vm_uuid"])
operation["service_was_running_hint"] = before.get("state") == "running"
# This write is a prerequisite for stopping the guest. Propagate any
# failure so the child endpoint is never called without a durable hint.
await asyncio.to_thread(
self.store.set, "operation", operation["operation_id"],
json.dumps(operation))
async def _check_absent(self, operation):
operation["safe_to_refund"] = False
components = self.registry.components.get(operation["host_id"])
profile = components.manager.current() if components else None
if profile and not profile.get("key_locked"):
try:
absent = await asyncio.to_thread(self._service(profile).provisioning_absent,
operation["vm_uuid"], operation["vm_name"],
uuid_resources=operation.get("resource_naming") == "uuid-v1",
legacy_name_collision=(
operation.get("resource_naming") is None
and operation.get("error") == "虚拟机名称已存在"
))
if absent:
await asyncio.to_thread(self._cleanup_absent_records, components,
profile, operation["vm_uuid"])
operation.update(status="failed", safe_to_refund=True)
except Exception:
pass
@staticmethod
def _cleanup_absent_records(components, profile, vm_id):
from .metrics import host_scope
scope = host_scope(profile)
components.state.users.delete_vm_owner(scope, vm_id)
components.store.delete("vm_credential", f"{scope}.{vm_id}")
prefix = scope + "."
for key, raw in components.store.list("ip_claim").items():
if key.startswith(prefix) and json.loads(raw).get("vm_uuid") == vm_id:
components.store.delete("ip_claim", key)
def _public_operation(self, record):
result = {key: record[key] for key in (
"operation_id", "status", "result", "error", "safe_to_refund",
"service_was_running_hint", "created_at", "finished_at")
if key in record}
# This is deliberately live process state, not a persisted claim. PHP
# must not close a stale policy job while its original executor can
# still mutate libvirt. It retries after the executor becomes terminal.
result["executor_active"] = record.get("operation_id") in self.active_operations
return result
@staticmethod
def _service(profile):
from .libvirt_service import VMService
return VMService(profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"],
profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""))
async def _ticket(self, host_id, vm_id, delegated, send):
if not delegated.get("session_id"):
raise EngineAuthError("控制台需要有效的网站会话", 403)
components = self.registry.components[host_id]
profile = components.manager.current()
if profile is None or profile.get("key_locked"):
raise EngineAuthError("宿主机尚未就绪", 409)
from .metrics import host_scope
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
raise EngineAuthError("虚拟机不存在", 404)
await asyncio.to_thread(self._service(profile).console_port, vm_id)
if not await self._session_active(delegated, host_id, vm_id):
raise EngineAuthError("网站会话已失效", 403)
ticket = secrets.token_urlsafe(32)
key = hashlib.sha256(ticket.encode("ascii")).hexdigest()
record = {"host_id": host_id, "vm_id": vm_id, "actor": delegated, "expires_at": int(time.time()) + 60}
await asyncio.to_thread(self.store.insert, "console_ticket", key, json.dumps(record))
await json_response(send, 200, {"ticket": ticket, "expires_in": 60, "websocket_path": "/engine-console"})
async def _session_active(self, delegated, host_id, vm_id):
if self.introspection is not None:
return await self.introspection(delegated, host_id, vm_id)
return await asyncio.to_thread(self._introspect, delegated, host_id, vm_id)
def _introspect(self, delegated, host_id, vm_id):
url = self.config.get("php_introspection_url", "")
parsed = urlsplit(url)
if (parsed.scheme not in {"http", "https"} or parsed.hostname not in {"127.0.0.1", "::1", "localhost"}
or parsed.username or parsed.password or parsed.fragment):
return False
body = json.dumps({"session_id": delegated["session_id"], "user_id": delegated["user_id"],
"host_id": host_id, "vm_uuid": vm_id}, separators=(",", ":")).encode()
secret = decode_secret(self.config.get("introspection_secret", self.config["service_secret"]))
headers = signed_headers(secret, "POST", parsed.path + ("?" + parsed.query if parsed.query else ""), body, delegated)
headers["Content-Type"] = "application/json"
if self.config.get("php_introspection_host"):
headers["Host"] = self.config["php_introspection_host"]
try:
opener = build_opener(ProxyHandler({}), NoRedirects())
with opener.open(URLRequest(url, body, headers, method="POST"), timeout=5) as response:
raw = response.read(8193)
return response.status == 200 and len(raw) <= 8192 and json.loads(raw).get("active") is True
except Exception:
return False
async def _console(self, scope, receive, send):
from starlette.websockets import WebSocket
socket = WebSocket(scope, receive=receive, send=send)
if scope["path"] != "/engine-console" or scope.get("scheme") != "wss":
await socket.close(code=4403)
return
from .app import _origin_matches_host
if not _origin_matches_host(socket.headers.get("origin"), socket.headers.get("host")):
await socket.close(code=4403)
return
protocols = [value for value in scope.get("subprotocols", []) if value.startswith("cloud-ticket.")]
if len(protocols) != 1 or not re.fullmatch(r"cloud-ticket\.[A-Za-z0-9_-]{43}", protocols[0]):
await socket.close(code=4401)
return
key = hashlib.sha256(protocols[0][13:].encode("ascii")).hexdigest()
raw = await asyncio.to_thread(self.store.get, "console_ticket", key)
if raw is None:
await socket.close(code=4401)
return
record = json.loads(raw)
if record["expires_at"] < int(time.time()) or not await asyncio.to_thread(self.store.insert, "ticket_used", key, "1"):
await socket.close(code=4401)
return
await asyncio.to_thread(self.store.delete, "console_ticket", key)
delegated, host_id, vm_id = record["actor"], record["host_id"], record["vm_id"]
if not await self._session_active(delegated, host_id, vm_id):
await socket.close(code=4403)
return
token = TRUSTED_ACTOR.set(delegated)
relay = watcher = None
try:
await self.registry.get(host_id)
components = self.registry.components[host_id]
profile = components.manager.current()
from .metrics import host_scope
from .console import relay_console
if profile is None or profile.get("key_locked"):
await socket.close(code=4409)
return
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
await socket.close(code=4403)
return
port = await asyncio.to_thread(self._service(profile).console_port, vm_id)
await socket.accept(subprotocol=protocols[0])
async def watch():
while True:
await asyncio.sleep(self.console_check_interval)
if not await self._session_active(delegated, host_id, vm_id):
return
current = components.manager.current()
if current is None or current.get("key_locked"):
return
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(current), vm_id, delegated["user_id"]):
return
relay = asyncio.create_task(relay_console(socket, port=port,
ssh_target=profile["ssh_target"], ssh_binary="ssh",
known_hosts_file=profile["ssh_known_hosts_file"], ssh_config_file=profile["ssh_config_file"]))
watcher = asyncio.create_task(watch())
await asyncio.wait({relay, watcher}, return_when=asyncio.FIRST_COMPLETED)
except Exception:
LOG.warning("Engine console closed after transport/authorization failure")
finally:
for task in (relay, watcher):
if task is not None:
task.cancel()
await asyncio.gather(*(task for task in (relay, watcher) if task is not None), return_exceptions=True)
with suppress(Exception):
await socket.close(code=4403)
TRUSTED_ACTOR.reset(token)
async def _lifespan(self, receive, send):
while True:
message = await receive()
if message["type"] == "lifespan.startup":
try:
await self.registry.start()
# Remote cache GC must never hold engine readiness hostage:
# several offline SSH/libvirt hosts can exceed the updater's
# health-check window. Track cleanup in the background so
# update apply sees it as maintenance, and skip eager GC
# while an update job is still completing this restart.
if not await asyncio.to_thread(self.updater.active_job):
for host_id in tuple(self.registry.components):
self._background(
self._cleanup_host_user_iso_tombstones(host_id))
self._background(
self._cleanup_host_cloud_template_tombstones(
host_id))
await send({"type": "lifespan.startup.complete"})
except Exception:
LOG.exception("Engine startup failed")
await send({"type": "lifespan.startup.failed", "message": "引擎初始化失败"})
return
elif message["type"] == "lifespan.shutdown":
# A task awaiting asyncio.to_thread cannot stop the underlying
# libvirt call when cancelled. Drain these bounded cleanup
# tasks before closing host services so no hidden mutation can
# outlive the engine lifespan.
await asyncio.gather(
*tuple(self.maintenance_tasks), return_exceptions=True)
await self.registry.close()
await send({"type": "lifespan.shutdown.complete"})
return
def create_engine():
default = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
config = private_config(Path(os.environ.get("CLOUD_ENGINE_CONFIG", str(default))))
secret = decode_secret(config["service_secret"])
del secret
key = master_key(Path(config["master_key_file"]))
store = MainStore(config["main_database"], key)
logs = LogStore(config["log_database"], key)
return EngineApplication(config, store, logs)