You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1915 lines
95 KiB
Python
1915 lines
95 KiB
Python
"""Private PHP -> libvirt boundary. The legacy browser app is never exposed.
|
|
|
|
Only allowlisted VM/host/template resources are dispatched to per-host services.
|
|
The public console endpoint accepts a short-lived, one-use ticket instead of a
|
|
PHP cookie, and rechecks the PHP session while the console remains connected.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import errno
|
|
import hashlib
|
|
import hmac
|
|
import inspect
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import secrets
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from contextlib import AsyncExitStack, suppress
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qsl, urlsplit
|
|
from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLRequest,
|
|
build_opener)
|
|
|
|
|
|
from .cloud_templates import (
|
|
MAX_DOWNLOAD_SECONDS,
|
|
MAX_IMAGE_BYTES,
|
|
TemplateError,
|
|
template_volume_name,
|
|
validate_template_metadata,
|
|
validate_template_name,
|
|
verify_qcow2_image,
|
|
)
|
|
from .cloud_template_runtime import (
|
|
CACHE_TOMBSTONE_KIND,
|
|
MIN_DOWNLOAD_BYTES,
|
|
CloudTemplateRuntime,
|
|
)
|
|
from .engine_context import (EngineComponents, HostStore, LogStore, MainStore,
|
|
TRUSTED_ACTOR, actor_session)
|
|
from .engine_security import (EngineAuthError, authenticate, decode_secret,
|
|
signed_headers)
|
|
from .updater import UpdateError, Updater
|
|
from .user_isos import UserISORuntime
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
OP_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z")
|
|
VM_ID = r"[0-9a-fA-F-]{36}"
|
|
JSON_LIMIT = 1024 * 1024
|
|
CLOUD_UPLOAD_IDLE_SECONDS = 60
|
|
CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024
|
|
READ_PATHS = (
|
|
r"host", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms",
|
|
r"vm-states", r"vm-metadata",
|
|
r"user-isos/[0-9a-f]{32}",
|
|
rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue)?",
|
|
r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+",
|
|
)
|
|
WRITE_PATHS = {
|
|
"POST": (r"host/(?:scan|unlock|test|discover-vms)",
|
|
r"vm-templates/(?:import|upload)", r"rescue-templates",
|
|
r"user-isos/import",
|
|
r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|rescue|console-ticket)",
|
|
rf"vms/{VM_ID}/(?:service|network)-(?:suspend|resume)",
|
|
rf"vms/{VM_ID}/rescue/(?:exit|force-start)", r"reinstall-jobs/[A-Za-z0-9_-]+/ack-password"),
|
|
"PUT": (r"host",),
|
|
"PATCH": (r"vm-templates/[A-Za-z0-9_-]+",),
|
|
"DELETE": (rf"vms/{VM_ID}", r"vm-templates/[A-Za-z0-9_-]+",
|
|
r"rescue-templates/[A-Za-z0-9_-]+",
|
|
r"user-isos/[0-9a-f]{32}"),
|
|
}
|
|
|
|
|
|
class _CloudUploadWriter:
|
|
"""Own a temporary upload descriptor used only from worker threads."""
|
|
|
|
def __init__(self, directory: Path) -> None:
|
|
descriptor, raw_path = tempfile.mkstemp(
|
|
prefix=".cloud-image-upload-",
|
|
suffix=".qcow2",
|
|
dir=directory,
|
|
)
|
|
try:
|
|
os.fchmod(descriptor, 0o600)
|
|
except BaseException:
|
|
os.close(descriptor)
|
|
Path(raw_path).unlink(missing_ok=True)
|
|
raise
|
|
self.path = Path(raw_path)
|
|
self._descriptor = descriptor
|
|
self._lock = threading.Lock()
|
|
|
|
def write(self, chunk: memoryview) -> None:
|
|
"""Write the complete bounded chunk before accepting another one."""
|
|
with self._lock:
|
|
if self._descriptor < 0:
|
|
raise OSError(errno.EBADF, "cloud upload is closed")
|
|
remaining = chunk
|
|
while remaining:
|
|
written = os.write(self._descriptor, remaining)
|
|
if written <= 0:
|
|
raise OSError(errno.EIO, "cloud upload write made no progress")
|
|
remaining = remaining[written:]
|
|
|
|
def sync_and_close(self) -> None:
|
|
"""Durably flush and close the descriptor before image validation."""
|
|
with self._lock:
|
|
if self._descriptor < 0:
|
|
return
|
|
descriptor = self._descriptor
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
try:
|
|
os.close(descriptor)
|
|
finally:
|
|
self._descriptor = -1
|
|
|
|
def close(self) -> None:
|
|
"""Close after an interrupted request, waiting for an in-flight write."""
|
|
with self._lock:
|
|
if self._descriptor < 0:
|
|
return
|
|
descriptor = self._descriptor
|
|
try:
|
|
os.close(descriptor)
|
|
finally:
|
|
self._descriptor = -1
|
|
|
|
|
|
class NoRedirects(HTTPRedirectHandler):
|
|
def redirect_request(self, *args, **kwargs):
|
|
raise ValueError("Session introspection redirects are not allowed")
|
|
|
|
|
|
class UpdateServiceUncertain(UpdateError):
|
|
"""The worker could not be confirmed stopped, so its job must stay active."""
|
|
|
|
|
|
def private_config(path: Path) -> dict:
|
|
info = path.lstat()
|
|
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 or info.st_uid != os.geteuid():
|
|
raise ValueError("引擎配置必须为运行用户拥有的普通 0600 文件")
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
try:
|
|
raw = os.read(descriptor, JSON_LIMIT + 1)
|
|
finally:
|
|
os.close(descriptor)
|
|
if len(raw) > JSON_LIMIT:
|
|
raise ValueError("引擎配置过大")
|
|
value = json.loads(raw)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("引擎配置无效")
|
|
return value
|
|
|
|
|
|
def master_key(path: Path) -> bytes:
|
|
from .database_bootstrap import DatabaseBootstrap
|
|
if not path.is_absolute():
|
|
raise ValueError("主密钥路径必须是绝对路径")
|
|
bootstrap = DatabaseBootstrap(path.parent)
|
|
bootstrap.key_file = path
|
|
return bootstrap.master_key(create=True)
|
|
|
|
|
|
class HostRegistry:
|
|
def __init__(self, config, store, log_store, *, factory=None, user_isos=None,
|
|
cloud_templates=None):
|
|
self.config = config
|
|
self.store = store
|
|
self.log_store = log_store
|
|
self.factory = factory
|
|
self.user_isos = user_isos or UserISORuntime(
|
|
store, Path(config["data_dir"]))
|
|
self._owns_user_isos = user_isos is None
|
|
self.cloud_templates = cloud_templates or CloudTemplateRuntime(
|
|
store, Path(config["data_dir"]))
|
|
self._owns_cloud_templates = cloud_templates is None
|
|
self.components: dict[str, EngineComponents] = {}
|
|
self.apps = {}
|
|
# Constructed by synchronous factories before an event loop may exist.
|
|
# Bind the lock lazily inside the ASGI lifespan/request loop.
|
|
self.lock: asyncio.Lock | None = None
|
|
self.lifespans = AsyncExitStack()
|
|
|
|
async def start(self):
|
|
if self.lock is None:
|
|
self.lock = asyncio.Lock()
|
|
await asyncio.to_thread(self.user_isos.restore_jobs)
|
|
await asyncio.to_thread(self.cloud_templates.restore_jobs)
|
|
await self.lifespans.__aenter__()
|
|
for host_id in await asyncio.to_thread(self.store.list, "engine_host"):
|
|
await self.get(host_id)
|
|
|
|
async def close(self):
|
|
await self.lifespans.aclose()
|
|
if self._owns_user_isos:
|
|
await asyncio.to_thread(self.user_isos.close)
|
|
if self._owns_cloud_templates:
|
|
await asyncio.to_thread(self.cloud_templates.close)
|
|
|
|
async def get(self, host_id: str):
|
|
if not HOST_ID.fullmatch(host_id):
|
|
raise EngineAuthError("宿主机标识无效", 400)
|
|
if self.lock is None:
|
|
self.lock = asyncio.Lock()
|
|
async with self.lock:
|
|
if host_id in self.apps:
|
|
return self.apps[host_id]
|
|
from .app import create_app
|
|
from .config import Settings
|
|
directory = Path(self.config["data_dir"]) / "hosts" / host_id
|
|
if self.factory is not None:
|
|
try:
|
|
inspect.signature(self.factory).bind(
|
|
host_id, self.user_isos, self.cloud_templates)
|
|
except TypeError:
|
|
try:
|
|
inspect.signature(self.factory).bind(host_id, self.user_isos)
|
|
except TypeError:
|
|
components, app = self.factory(host_id)
|
|
else:
|
|
components, app = self.factory(host_id, self.user_isos)
|
|
else:
|
|
components, app = self.factory(
|
|
host_id, self.user_isos, self.cloud_templates)
|
|
else:
|
|
components = await asyncio.to_thread(EngineComponents.for_host,
|
|
self.store, self.log_store, host_id, directory,
|
|
self.user_isos, self.cloud_templates)
|
|
settings = Settings(data_dir=directory)
|
|
app = await asyncio.to_thread(create_app, settings, engine_components=components)
|
|
self.components[host_id] = components
|
|
await self.lifespans.enter_async_context(app.router.lifespan_context(app))
|
|
self.apps[host_id] = app
|
|
return app
|
|
|
|
|
|
async def json_response(send, status: int, payload):
|
|
body = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode()
|
|
await send({"type": "http.response.start", "status": status,
|
|
"headers": [(b"content-type", b"application/json"), (b"cache-control", b"no-store")]})
|
|
await send({"type": "http.response.body", "body": body})
|
|
|
|
|
|
def raw_target(scope) -> str:
|
|
path = scope.get("raw_path", scope["path"].encode("ascii")).decode("ascii")
|
|
query = scope.get("query_string", b"")
|
|
return path + ("?" + query.decode("ascii") if query else "")
|
|
|
|
|
|
def request_headers(scope):
|
|
values = {}
|
|
for key, value in scope.get("headers", []):
|
|
name = key.decode("ascii").lower()
|
|
if name.startswith("x-cloud-") and name in values:
|
|
raise EngineAuthError("重复的内部认证头")
|
|
values[name] = value.decode("latin-1")
|
|
return values
|
|
|
|
|
|
class EngineApplication:
|
|
def __init__(self, config, store, log_store, *, registry=None, introspection=None,
|
|
updater=None, update_trigger=None):
|
|
self.config = config
|
|
self.store = store
|
|
self.secret = decode_secret(config["service_secret"])
|
|
self.registry = registry or HostRegistry(config, store, log_store)
|
|
self.introspection = introspection
|
|
project_dir = Path(config.get("project_dir", Path(__file__).resolve().parents[2]))
|
|
self.updater = updater or Updater(
|
|
project_dir, store, worker_active=self._update_service_active)
|
|
self.update_trigger = update_trigger or self._start_update_service
|
|
# Serializes the transition into update maintenance mode with every
|
|
# request that can mutate libvirt or durable operation state. The
|
|
# lock is created lazily so synchronous test/application factories do
|
|
# not bind it to an event loop that has not started yet.
|
|
self.update_coordination_lock: asyncio.Lock | None = None
|
|
self.active_operations: set[str] = set()
|
|
self.maintenance_tasks: set[asyncio.Task] = set()
|
|
# Target-state operations are idempotent individually, but two
|
|
# different operation IDs can request opposite states at the same
|
|
# time (for example traffic suspension ending at a period boundary).
|
|
# Serialize those mutations per VM so the later request always wins
|
|
# on the hypervisor as well as in the billing database.
|
|
self.target_locks: dict[tuple[str, str], asyncio.Lock] = {}
|
|
self.console_check_interval = 15
|
|
|
|
def _background(self, coroutine) -> None:
|
|
task = asyncio.create_task(coroutine)
|
|
self.maintenance_tasks.add(task)
|
|
task.add_done_callback(self.maintenance_tasks.discard)
|
|
|
|
def _target_lock(self, host_id: str, vm_id: str) -> asyncio.Lock:
|
|
key = (host_id, vm_id)
|
|
lock = self.target_locks.get(key)
|
|
if lock is None:
|
|
lock = asyncio.Lock()
|
|
self.target_locks[key] = lock
|
|
return lock
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope["type"] == "lifespan":
|
|
await self._lifespan(receive, send)
|
|
return
|
|
if scope["type"] == "websocket":
|
|
try:
|
|
await self._console(scope, receive, send)
|
|
except Exception:
|
|
LOG.warning("Engine console rejected because a dependency failed")
|
|
await send({"type": "websocket.close", "code": 4403})
|
|
return
|
|
if scope["type"] != "http":
|
|
return
|
|
token = None
|
|
response_started = False
|
|
async def tracked_send(message):
|
|
nonlocal response_started
|
|
if message["type"] == "http.response.start":
|
|
response_started = True
|
|
await send(message)
|
|
try:
|
|
# Public reverse proxies must never forward these routes. Reject
|
|
# non-loopback peers even when they have spoofed forwarded headers.
|
|
client = scope.get("client")
|
|
if client and client[0] not in {"127.0.0.1", "::1", "testclient"}:
|
|
raise EngineAuthError("内部引擎仅接受本机服务请求", 403)
|
|
headers = request_headers(scope)
|
|
target = raw_target(scope)
|
|
upload_resource = None
|
|
if scope["method"] == "POST":
|
|
if scope["path"] == "/internal/v1/vm-templates/upload":
|
|
upload_resource = "vm-templates/upload"
|
|
elif re.fullmatch(
|
|
r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates",
|
|
scope["path"]):
|
|
upload_resource = "rescue-templates"
|
|
upload = upload_resource is not None
|
|
if upload:
|
|
digest = headers.get("x-cloud-content-sha256", "")
|
|
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
|
|
scope["method"], target, digest, self.store)
|
|
self._authorize_route(
|
|
scope["method"], upload_resource, delegated)
|
|
checksum = hashlib.sha256()
|
|
validation = {"complete": False, "valid": False}
|
|
scope = {
|
|
**scope,
|
|
"engine_upload_validation": validation,
|
|
"engine_upload_digest": digest,
|
|
}
|
|
async def verified_receive():
|
|
message = await receive()
|
|
if message["type"] == "http.request":
|
|
chunk = message.get("body", b"")
|
|
checksum.update(chunk)
|
|
if not message.get("more_body", False):
|
|
validation.update(complete=True,
|
|
valid=hmac.compare_digest(checksum.hexdigest(), digest))
|
|
return message
|
|
child_receive = verified_receive
|
|
body = None
|
|
else:
|
|
chunks = []
|
|
size = 0
|
|
while True:
|
|
message = await receive()
|
|
if message["type"] == "http.disconnect":
|
|
return
|
|
chunk = message.get("body", b"")
|
|
size += len(chunk)
|
|
if size > JSON_LIMIT:
|
|
raise EngineAuthError("内部请求超过 1 MiB", 413)
|
|
chunks.append(chunk)
|
|
if not message.get("more_body", False):
|
|
break
|
|
body = b"".join(chunks)
|
|
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
|
|
scope["method"], target, hashlib.sha256(body).hexdigest(), self.store)
|
|
delivered = False
|
|
async def replay_receive():
|
|
nonlocal delivered
|
|
if not delivered:
|
|
delivered = True
|
|
return {"type": "http.request", "body": body, "more_body": False}
|
|
return await receive()
|
|
child_receive = replay_receive
|
|
token = TRUSTED_ACTOR.set(delegated)
|
|
await self._dispatch(scope, child_receive, tracked_send, body, delegated)
|
|
except EngineAuthError as exc:
|
|
if not response_started:
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
except UpdateError as exc:
|
|
if not response_started:
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
except Exception:
|
|
LOG.exception("Private engine request failed")
|
|
if not response_started:
|
|
await json_response(send, 503, {"error": "引擎服务暂时不可用,请检查引擎日志"})
|
|
finally:
|
|
if token is not None:
|
|
TRUSTED_ACTOR.reset(token)
|
|
|
|
@staticmethod
|
|
def _authorize_route(method: str, resource: str, delegated: dict):
|
|
paths = READ_PATHS if method == "GET" else WRITE_PATHS.get(method, ())
|
|
if not any(re.fullmatch(pattern, resource) for pattern in paths):
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
administration = (resource.startswith("host") or resource.startswith("vm-templates/")
|
|
or resource.startswith("rescue-templates/"))
|
|
administration_write = method != "GET" and (administration or resource == "rescue-templates")
|
|
if administration and not delegated["is_admin"]:
|
|
raise EngineAuthError("仅管理员可访问此资源", 403)
|
|
if method != "GET" and not delegated["can_manage"]:
|
|
raise EngineAuthError("当前账户不允许操作", 403)
|
|
if administration_write and not delegated["is_super_admin"]:
|
|
raise EngineAuthError("仅超级管理员可修改宿主机和模版", 403)
|
|
|
|
async def _dispatch(self, scope, receive, send, body, delegated):
|
|
path = scope["path"]
|
|
if path.startswith("/internal/v1/update/"):
|
|
await self._update_dispatch(scope["method"], path, body, delegated, send)
|
|
return
|
|
method = scope["method"]
|
|
operation_reconcile = (
|
|
method == "GET"
|
|
and re.fullmatch(r"/internal/v1/operations/[A-Za-z0-9_-]{8,80}", path) is not None
|
|
and scope.get("query_string", b"") != b"reconcile=false"
|
|
)
|
|
# A browser upload can spend minutes waiting for request-body chunks.
|
|
# Its handler acquires the update coordination lock only to reserve and
|
|
# finally queue the import, so unrelated VM writes remain available.
|
|
if method == "POST" and path == "/internal/v1/vm-templates/upload":
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
return
|
|
if method != "GET" or operation_reconcile:
|
|
async with self._update_lock():
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
return
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
|
|
def _update_lock(self) -> asyncio.Lock:
|
|
if self.update_coordination_lock is None:
|
|
self.update_coordination_lock = asyncio.Lock()
|
|
return self.update_coordination_lock
|
|
|
|
async def _dispatch_regular(self, scope, receive, send, body, delegated):
|
|
path = scope["path"]
|
|
if scope["method"] == "POST" and path == "/internal/v1/hosts/scan":
|
|
if not delegated["is_super_admin"] or not delegated["can_manage"]:
|
|
raise EngineAuthError("仅超级管理员可以扫描宿主机", 403)
|
|
from .app import HostScanBody
|
|
from .remote_host import RemoteHostManager
|
|
from pydantic import ValidationError
|
|
try:
|
|
value = HostScanBody.model_validate_json(body)
|
|
except ValidationError as exc:
|
|
raise EngineAuthError("宿主机扫描参数无效", 422) from exc
|
|
manager = await asyncio.to_thread(RemoteHostManager,
|
|
Path(self.config["data_dir"]) / "discovery")
|
|
try:
|
|
result = await asyncio.to_thread(manager.scan, value.host, value.port)
|
|
await json_response(send, 200, result)
|
|
finally:
|
|
await asyncio.to_thread(manager.close)
|
|
return
|
|
if scope["method"] == "GET" and path == "/internal/v1/health":
|
|
await asyncio.to_thread(self.store.get, "engine", "readiness")
|
|
await json_response(send, 200, {"status": "ok"})
|
|
return
|
|
cloud_template_match = re.fullmatch(
|
|
r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|"
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
|
|
if cloud_template_match is not None:
|
|
resource = cloud_template_match[1]
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
await self._cloud_template_dispatch(
|
|
scope["method"], resource, body, send,
|
|
scope=scope, receive=receive)
|
|
return
|
|
user_iso_match = re.fullmatch(
|
|
r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|"
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
|
|
if user_iso_match is not None:
|
|
resource = user_iso_match[1]
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
await self._user_iso_dispatch(
|
|
scope["method"], resource, body, delegated, send)
|
|
return
|
|
op_match = re.fullmatch(r"/internal/v1/operations/([A-Za-z0-9_-]{8,80})", path)
|
|
if scope["method"] == "GET" and op_match:
|
|
query = scope.get("query_string", b"")
|
|
if query not in {b"", b"reconcile=false"}:
|
|
raise EngineAuthError("操作查询参数无效", 400)
|
|
operation = await self._operation(
|
|
op_match[1], delegated, reconcile=query != b"reconcile=false")
|
|
await json_response(send, 200, self._public_operation(operation))
|
|
return
|
|
match = re.fullmatch(r"/internal/v1/hosts/([A-Za-z0-9][A-Za-z0-9_-]{0,63})/(.+)", path)
|
|
if match is None:
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
host_id, resource = match.groups()
|
|
if re.fullmatch(
|
|
r"user-isos(?:/import|/[0-9a-f]{32}|"
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?", resource):
|
|
# User ISO ownership, quota, source bytes and deletion tombstones
|
|
# are engine-global. Keeping the historical host-prefixed alias
|
|
# reachable would let DELETE bypass replica GC and make the same
|
|
# catalog appear to have host-local semantics.
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
app = await self.registry.get(host_id)
|
|
if scope["method"] == "GET":
|
|
# Reads normally bypass the update coordination lock, but lazy
|
|
# cache GC mutates libvirt. Coordinate that maintenance work with
|
|
# online update and skip it while an update is already active.
|
|
async with self._update_lock():
|
|
if not await asyncio.to_thread(self.updater.active_job):
|
|
await self._cleanup_host_user_iso_tombstones(host_id)
|
|
await self._cleanup_host_cloud_template_tombstones(host_id)
|
|
else:
|
|
# Every host write reached this method while holding _update_lock
|
|
# and after the active-update gate in _dispatch().
|
|
await self._cleanup_host_user_iso_tombstones(host_id)
|
|
await self._cleanup_host_cloud_template_tombstones(host_id)
|
|
if resource.endswith("/console-ticket"):
|
|
vm_id = resource.split("/")[1]
|
|
await self._ticket(host_id, vm_id, delegated, send)
|
|
return
|
|
operation = None
|
|
create = scope["method"] == "POST" and resource == "vms"
|
|
mutation = scope["method"] == "DELETE" and resource.startswith("vms/") or (
|
|
scope["method"] == "POST" and bool(re.fullmatch(
|
|
rf"vms/{VM_ID}/(?:actions|reinstall|"
|
|
rf"(?:service|network)-(?:suspend|resume)|"
|
|
rf"rescue(?:/exit|/force-start)?)",
|
|
resource)))
|
|
if create or mutation:
|
|
if not OP_ID.fullmatch(delegated.get("operation_id", "")):
|
|
raise EngineAuthError("虚拟机操作缺少持久幂等编号", 403)
|
|
if create and not delegated.get("can_provision"):
|
|
raise EngineAuthError("虚拟机只能通过已确认套餐订单开通", 403)
|
|
operation, fresh = await self._reserve_operation(host_id, delegated, body,
|
|
resource=resource, method=scope["method"], target=raw_target(scope))
|
|
if operation.get("prevalidation_failed"):
|
|
await json_response(send, 422, self._public_operation(operation))
|
|
return
|
|
if not fresh:
|
|
if (operation["status"] in {"running", "pending", "uncertain"}
|
|
and self._service_target_method(operation.get("resource", ""))):
|
|
operation = await self._mutation_operation(operation)
|
|
if operation["status"] == "succeeded":
|
|
await json_response(send, 201 if create else 200,
|
|
{**operation["result"], **self._public_operation(operation)})
|
|
else:
|
|
await json_response(send, 202, self._public_operation(operation))
|
|
return
|
|
if create:
|
|
delegated["vm_uuid"] = operation["vm_uuid"]
|
|
self.active_operations.add(operation["operation_id"])
|
|
child_scope = {**scope, "path": "/api/v1/" + resource,
|
|
"raw_path": ("/api/v1/" + resource).encode(), "root_path": ""}
|
|
if operation is None:
|
|
# Register before saving so all persisted hosts are restored for
|
|
# unattended sampling after restart, even if the HTTP caller dies.
|
|
if resource == "host" and scope["method"] == "PUT":
|
|
await asyncio.to_thread(self.store.set, "engine_host", host_id, json.dumps({"host_id": host_id}))
|
|
await app(child_scope, receive, send)
|
|
return
|
|
messages = []
|
|
async def capture(message):
|
|
messages.append(message)
|
|
try:
|
|
target_method = self._service_target_method(operation.get("resource", ""))
|
|
# Every VM mutation shares the same lock, not just billing target
|
|
# state. A start/reset/delete that timed out at PHP may still be
|
|
# executing here when expiry enforcement arrives. Serializing the
|
|
# later policy call guarantees that its desired state wins. The
|
|
# reinstall/rescue endpoints register their per-VM pending guard
|
|
# before returning 202; child endpoints reject every other
|
|
# mutation until those background jobs remove that guard.
|
|
async with self._target_lock(host_id, operation["vm_uuid"]):
|
|
if target_method == "service_suspend":
|
|
await self._capture_service_was_running_hint(
|
|
host_id, operation, delegated)
|
|
await app(child_scope, receive, capture)
|
|
status = next(item["status"] for item in messages if item["type"] == "http.response.start")
|
|
payload = json.loads(b"".join(item.get("body", b"") for item in messages
|
|
if item["type"] == "http.response.body"))
|
|
operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else (
|
|
"failed" if not create and status < 500 else "uncertain")
|
|
if operation["status"] == "succeeded":
|
|
operation["result"] = payload
|
|
elif operation["status"] == "pending":
|
|
operation["result"] = payload
|
|
operation["job_id"] = payload["job_id"]
|
|
else:
|
|
operation["error"] = payload.get("detail") or payload.get("error") or "虚拟机开通失败"
|
|
if create:
|
|
await self._check_absent(operation)
|
|
operation["finished_at"] = int(time.time())
|
|
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
|
|
if operation["status"] in {"succeeded", "pending"}:
|
|
await json_response(send, status, {**payload, **self._public_operation(operation)})
|
|
else:
|
|
await json_response(send, status, {**payload, "operation_id": operation["operation_id"],
|
|
"status": operation["status"], "safe_to_refund": operation["safe_to_refund"]})
|
|
except BaseException:
|
|
operation["status"] = "uncertain"
|
|
operation["error"] = "开通结果不确定,请查询操作状态,不要重复创建"
|
|
with suppress(Exception):
|
|
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
|
|
raise
|
|
finally:
|
|
self.active_operations.discard(operation["operation_id"])
|
|
|
|
@staticmethod
|
|
def _cloud_upload_metadata(scope) -> tuple[str, str, str]:
|
|
query = scope.get("query_string", b"")
|
|
if (not isinstance(query, bytes) or len(query) > 2048
|
|
or re.search(rb"%(?![0-9A-Fa-f]{2})", query) is not None):
|
|
raise TemplateError("云镜像上传参数无效", 422)
|
|
try:
|
|
pairs = parse_qsl(
|
|
query.decode("ascii"),
|
|
keep_blank_values=True,
|
|
strict_parsing=True,
|
|
max_num_fields=3,
|
|
errors="strict",
|
|
)
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
raise TemplateError("云镜像上传参数无效", 422) from exc
|
|
if (len(pairs) != 3
|
|
or {key for key, _value in pairs}
|
|
!= {"name", "os_family", "os_version"}):
|
|
raise TemplateError("云镜像上传参数无效", 422)
|
|
values = dict(pairs)
|
|
name = validate_template_name(values["name"])
|
|
os_family, os_version = validate_template_metadata(
|
|
values["os_family"], values["os_version"])
|
|
return name, os_family, os_version
|
|
|
|
@staticmethod
|
|
def _single_upload_header(scope, name: bytes) -> str | None:
|
|
values = [value.decode("latin-1")
|
|
for key, value in scope.get("headers", [])
|
|
if key.lower() == name]
|
|
if len(values) > 1:
|
|
raise TemplateError("云镜像上传请求头重复", 400)
|
|
return values[0] if values else None
|
|
|
|
async def _upload_cloud_template(self, scope, receive, send) -> None:
|
|
runtime = self.registry.cloud_templates
|
|
name, os_family, os_version = self._cloud_upload_metadata(scope)
|
|
content_type = self._single_upload_header(scope, b"content-type")
|
|
if content_type is None or content_type.lower() != "application/octet-stream":
|
|
raise TemplateError(
|
|
"云镜像上传必须使用 application/octet-stream", 415)
|
|
content_encoding = self._single_upload_header(scope, b"content-encoding")
|
|
if content_encoding is not None and content_encoding.lower() != "identity":
|
|
raise TemplateError("云镜像上传内容不能压缩", 415)
|
|
|
|
declared_raw = self._single_upload_header(scope, b"content-length")
|
|
if declared_raw is not None and re.fullmatch(r"[0-9]+", declared_raw) is None:
|
|
raise TemplateError("云镜像上传长度无效", 400)
|
|
declared = int(declared_raw) if declared_raw is not None else None
|
|
if declared is not None and declared < MIN_DOWNLOAD_BYTES:
|
|
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
|
|
if declared is not None and declared > MAX_IMAGE_BYTES:
|
|
raise TemplateError("云镜像超过 4 GiB 限制", 413)
|
|
|
|
async with self._update_lock():
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
raise EngineAuthError(
|
|
"面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
reservation = await asyncio.to_thread(
|
|
runtime.reserve_upload, name, os_family, os_version)
|
|
reservation_id = reservation["reservation_id"]
|
|
maximum = reservation["max_bytes"]
|
|
if declared is not None and declared > maximum:
|
|
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
|
|
raise TemplateError("面板云镜像持久目录空间不足", 507)
|
|
|
|
image_path: Path | None = None
|
|
handed_off = False
|
|
writer: _CloudUploadWriter | None = None
|
|
try:
|
|
writer = await asyncio.to_thread(
|
|
_CloudUploadWriter, reservation["directory"])
|
|
image_path = writer.path
|
|
checksum = hashlib.sha256()
|
|
size = 0
|
|
deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS
|
|
try:
|
|
while True:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TemplateError("云镜像上传超时", 408)
|
|
try:
|
|
message = await asyncio.wait_for(
|
|
receive(),
|
|
timeout=min(CLOUD_UPLOAD_IDLE_SECONDS, remaining),
|
|
)
|
|
except TimeoutError as exc:
|
|
raise TemplateError("云镜像上传超时", 408) from exc
|
|
if message["type"] == "http.disconnect":
|
|
raise TemplateError("云镜像上传已中断", 400)
|
|
if message["type"] != "http.request":
|
|
raise TemplateError("云镜像上传请求无效", 400)
|
|
chunk = message.get("body", b"")
|
|
size += len(chunk)
|
|
if declared is not None and size > declared:
|
|
raise TemplateError("云镜像上传内容超过声明长度", 400)
|
|
if size > maximum:
|
|
if maximum < MAX_IMAGE_BYTES:
|
|
raise TemplateError(
|
|
"面板云镜像持久目录空间不足", 507)
|
|
raise TemplateError("云镜像超过 4 GiB 限制", 413)
|
|
view = memoryview(chunk)
|
|
for offset in range(0, len(view), CLOUD_UPLOAD_WRITE_BYTES):
|
|
if time.monotonic() >= deadline:
|
|
raise TemplateError("云镜像上传超时", 408)
|
|
part = view[offset:offset + CLOUD_UPLOAD_WRITE_BYTES]
|
|
await asyncio.to_thread(writer.write, part)
|
|
checksum.update(part)
|
|
if not message.get("more_body", False):
|
|
break
|
|
await asyncio.to_thread(writer.sync_and_close)
|
|
except OSError as exc:
|
|
if exc.errno in {errno.ENOSPC, errno.EDQUOT}:
|
|
raise TemplateError(
|
|
"面板云镜像持久目录空间不足", 507) from exc
|
|
raise
|
|
if declared is not None and size != declared:
|
|
raise TemplateError("云镜像上传内容不完整", 400)
|
|
if size < MIN_DOWNLOAD_BYTES:
|
|
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
|
|
validation = scope.get("engine_upload_validation", {})
|
|
expected_digest = scope.get("engine_upload_digest", "")
|
|
digest = checksum.hexdigest()
|
|
if (not validation.get("complete")
|
|
or not validation.get("valid")
|
|
or not hmac.compare_digest(digest, expected_digest)):
|
|
raise EngineAuthError("云镜像摘要与签名不匹配", 401)
|
|
virtual_size = await asyncio.to_thread(
|
|
verify_qcow2_image, image_path)
|
|
# Recheck the update state at the commit boundary. An upload
|
|
# admitted before a service transition must not enqueue work
|
|
# against code that is being replaced.
|
|
async with self._update_lock():
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
raise EngineAuthError(
|
|
"面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
result = await asyncio.to_thread(
|
|
runtime.queue_upload,
|
|
reservation_id,
|
|
image_path,
|
|
digest,
|
|
virtual_size,
|
|
)
|
|
handed_off = True
|
|
await json_response(send, 202, result)
|
|
finally:
|
|
if not handed_off:
|
|
if writer is not None:
|
|
with suppress(OSError):
|
|
await asyncio.to_thread(writer.close)
|
|
if image_path is not None:
|
|
with suppress(OSError):
|
|
await asyncio.to_thread(image_path.unlink, missing_ok=True)
|
|
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
|
|
|
|
async def _cloud_template_dispatch(
|
|
self, method: str, resource: str, body: bytes | None, send,
|
|
*, scope=None, receive=None) -> None:
|
|
runtime = self.registry.cloud_templates
|
|
try:
|
|
if method == "GET" and resource == "vm-templates":
|
|
records = await asyncio.to_thread(runtime.list)
|
|
# A normal catalog read also repairs the narrow crash/lost-ack
|
|
# window between source deletion and its final tombstone flag.
|
|
# Once repaired, immediately resume cache GC on every loaded
|
|
# host instead of requiring an engine restart.
|
|
for host_id in tuple(self.registry.components):
|
|
self._background(
|
|
self._cleanup_host_cloud_template_tombstones(host_id))
|
|
await json_response(send, 200, {"templates": records})
|
|
return
|
|
if method == "POST" and resource == "vm-templates/import":
|
|
from .app import ImportTemplateBody
|
|
from pydantic import ValidationError
|
|
try:
|
|
value = ImportTemplateBody.model_validate_json(body or b"")
|
|
except ValidationError as exc:
|
|
raise TemplateError("云镜像导入参数无效", 422) from exc
|
|
result = await asyncio.to_thread(
|
|
runtime.queue_import, value.url, value.name, value.sha256)
|
|
await json_response(send, 202, result)
|
|
return
|
|
if method == "POST" and resource == "vm-templates/upload":
|
|
if scope is None or receive is None:
|
|
raise EngineAuthError("云镜像上传请求无效", 400)
|
|
await self._upload_cloud_template(scope, receive, send)
|
|
return
|
|
job_match = re.fullmatch(
|
|
r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource)
|
|
if method == "GET" and job_match is not None:
|
|
result = await asyncio.to_thread(
|
|
runtime.get_job, job_match[1])
|
|
await json_response(send, 200, result)
|
|
return
|
|
template_match = re.fullmatch(
|
|
r"vm-templates/([A-Za-z0-9_-]+)", resource)
|
|
if method == "PATCH" and template_match is not None:
|
|
from .app import RenameTemplateBody
|
|
from pydantic import ValidationError
|
|
try:
|
|
value = RenameTemplateBody.model_validate_json(body or b"")
|
|
except ValidationError as exc:
|
|
raise TemplateError("云镜像名称无效", 422) from exc
|
|
record = await asyncio.to_thread(
|
|
runtime.rename, template_match[1], value.name)
|
|
await json_response(send, 200, {"template": record})
|
|
return
|
|
if method == "DELETE" and template_match is not None:
|
|
from .app import DeleteTemplateBody
|
|
from pydantic import ValidationError
|
|
try:
|
|
value = DeleteTemplateBody.model_validate_json(body or b"")
|
|
except ValidationError as exc:
|
|
raise TemplateError("云镜像删除参数无效", 422) from exc
|
|
template_id = template_match[1]
|
|
tombstone_key, tombstone = await asyncio.to_thread(
|
|
self._delete_cloud_template_source,
|
|
template_id, value.created_at)
|
|
for host_id in tombstone["pending_hosts"]:
|
|
self._background(
|
|
self._cleanup_cloud_template_tombstone_on_host(
|
|
tombstone_key, tombstone, host_id))
|
|
await json_response(send, 200, {
|
|
"deleted": True,
|
|
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
|
|
})
|
|
return
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
except TemplateError as exc:
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
|
async def _user_iso_dispatch(self, method: str, resource: str,
|
|
body: bytes | None, delegated: dict, send) -> None:
|
|
runtime = self.registry.user_isos
|
|
user_id = delegated["user_id"]
|
|
try:
|
|
if method == "GET" and resource == "user-isos":
|
|
result = await asyncio.to_thread(runtime.list_for, user_id)
|
|
await json_response(send, 200, result)
|
|
return
|
|
if method == "POST" and resource == "user-isos/import":
|
|
from .app import ImportUserISOBody
|
|
from pydantic import ValidationError
|
|
try:
|
|
value = ImportUserISOBody.model_validate_json(body or b"")
|
|
except ValidationError as exc:
|
|
raise TemplateError("ISO 导入参数无效", 422) from exc
|
|
result = await asyncio.to_thread(
|
|
runtime.queue_import, user_id, value.url, value.name,
|
|
value.sha256)
|
|
await json_response(send, 202, result)
|
|
return
|
|
job_match = re.fullmatch(
|
|
r"user-isos/import-jobs/([A-Za-z0-9_-]+)", resource)
|
|
if method == "GET" and job_match is not None:
|
|
result = await asyncio.to_thread(
|
|
runtime.get_job_for, user_id, job_match[1])
|
|
await json_response(send, 200, result)
|
|
return
|
|
iso_match = re.fullmatch(r"user-isos/([0-9a-f]{32})", resource)
|
|
if iso_match is not None and method == "GET":
|
|
record = await asyncio.to_thread(
|
|
runtime.get_for, user_id, iso_match[1])
|
|
await json_response(
|
|
send, 200, {"iso": runtime.public_record(record)})
|
|
return
|
|
if iso_match is not None and method == "DELETE":
|
|
iso_id = iso_match[1]
|
|
tombstone_key, tombstone = await asyncio.to_thread(
|
|
self._delete_user_iso_source, user_id, iso_id)
|
|
for host_id in tombstone["pending_hosts"]:
|
|
self._background(self._cleanup_user_iso_tombstone_on_host(
|
|
tombstone_key, tombstone, host_id))
|
|
await json_response(send, 200, {
|
|
"deleted": True,
|
|
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
|
|
})
|
|
return
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
except TemplateError as exc:
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
|
def _delete_cloud_template_source(
|
|
self, template_id: str,
|
|
expected_created_at: int) -> tuple[str, dict]:
|
|
"""Commit global source deletion and durable per-host cache GC."""
|
|
runtime = self.registry.cloud_templates
|
|
with runtime.lock:
|
|
raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id)
|
|
if raw is not None:
|
|
tombstone = self._decode_cloud_template_tombstone(
|
|
template_id, raw)
|
|
if tombstone["template_created_at"] != expected_created_at:
|
|
raise TemplateError(
|
|
"云镜像版本已变化,请刷新后重试", 409)
|
|
if tombstone["source_deleted"]:
|
|
return template_id, tombstone
|
|
|
|
try:
|
|
record = runtime.get(template_id)
|
|
except TemplateError as exc:
|
|
if raw is None or exc.status != 404:
|
|
raise
|
|
tombstone["source_deleted"] = True
|
|
self.store.set(
|
|
CACHE_TOMBSTONE_KIND, template_id,
|
|
json.dumps(tombstone, separators=(",", ":")),
|
|
)
|
|
return template_id, tombstone
|
|
|
|
source_volume_name = record.get("volume_name")
|
|
if source_volume_name != template_volume_name(template_id):
|
|
raise TemplateError("云镜像目录记录损坏,无法安全删除", 500)
|
|
if record.get("created_at") != expected_created_at:
|
|
raise TemplateError("云镜像版本已变化,请刷新后重试", 409)
|
|
host_ids = sorted(
|
|
set(self.store.list("engine_host"))
|
|
| set(self.registry.components)
|
|
)
|
|
tombstone = {
|
|
"deletion_id": uuid.uuid4().hex,
|
|
"template_id": template_id,
|
|
"template_created_at": expected_created_at,
|
|
"source_volume_name": source_volume_name,
|
|
"pending_hosts": host_ids,
|
|
"source_deleted": False,
|
|
"created_at": int(time.time()),
|
|
}
|
|
encoded = json.dumps(tombstone, separators=(",", ":"))
|
|
self.store.set(CACHE_TOMBSTONE_KIND, template_id, encoded)
|
|
try:
|
|
# Re-enters runtime.lock, which is intentionally an RLock.
|
|
runtime.delete(template_id)
|
|
except Exception:
|
|
# A database DELETE acknowledgement can be lost before or
|
|
# after commit. The catalog alone is not sufficient proof of
|
|
# rollback because source unlink happens first. Reconcile both
|
|
# durable halves: only an intact catalog+source pair cancels
|
|
# this marker; a missing source completes retirement and keeps
|
|
# host GC durable.
|
|
if self.store.get(
|
|
CACHE_TOMBSTONE_KIND, template_id) == encoded:
|
|
reconciled = runtime._reconcile_cache_tombstone(
|
|
template_id, tombstone.copy())
|
|
if (reconciled is not None
|
|
and reconciled["source_deleted"]):
|
|
return template_id, reconciled
|
|
raise
|
|
tombstone["source_deleted"] = True
|
|
self.store.set(
|
|
CACHE_TOMBSTONE_KIND, template_id,
|
|
json.dumps(tombstone, separators=(",", ":")),
|
|
)
|
|
return template_id, tombstone
|
|
|
|
@staticmethod
|
|
def _decode_cloud_template_tombstone(key: str, raw: str) -> dict:
|
|
try:
|
|
value = json.loads(raw)
|
|
deletion_id = value["deletion_id"]
|
|
template_id = value["template_id"]
|
|
template_created_at = value["template_created_at"]
|
|
pending = value["pending_hosts"]
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise ValueError("cloud template tombstone is corrupt") from exc
|
|
try:
|
|
expected_volume_name = template_volume_name(template_id)
|
|
except TemplateError as exc:
|
|
raise ValueError("cloud template tombstone is corrupt") from exc
|
|
if (not isinstance(deletion_id, str)
|
|
or not re.fullmatch(r"[0-9a-f]{32}", deletion_id)
|
|
or key != template_id
|
|
or isinstance(template_created_at, bool)
|
|
or not isinstance(template_created_at, int)
|
|
or template_created_at < 1
|
|
or value.get("source_volume_name") != expected_volume_name
|
|
or not isinstance(pending, list)
|
|
or any(not isinstance(host_id, str)
|
|
or not HOST_ID.fullmatch(host_id)
|
|
for host_id in pending)
|
|
or len(set(pending)) != len(pending)
|
|
or not isinstance(value.get("source_deleted"), bool)):
|
|
raise ValueError("cloud template tombstone is corrupt")
|
|
return value
|
|
|
|
async def _cleanup_cloud_template_tombstone_on_host(
|
|
self, key: str, tombstone: dict, host_id: str) -> None:
|
|
if (not tombstone.get("source_deleted")
|
|
or host_id not in tombstone["pending_hosts"]):
|
|
return
|
|
try:
|
|
await self.registry.get(host_id)
|
|
components = self.registry.components[host_id]
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
if profile is None or profile.get("key_locked"):
|
|
return
|
|
service = self._service(profile)
|
|
await asyncio.to_thread(
|
|
self._cleanup_cloud_template_tombstone_on_host_sync,
|
|
key, tombstone["deletion_id"], host_id, service, components)
|
|
except Exception as exc:
|
|
LOG.info(
|
|
"Cloud template cache cleanup deferred: host=%s type=%s",
|
|
host_id, type(exc).__name__)
|
|
|
|
def _cleanup_cloud_template_tombstone_on_host_sync(
|
|
self, key: str, deletion_id: str, host_id: str,
|
|
service, components: EngineComponents) -> None:
|
|
"""Revalidate the deletion generation at the libvirt commit point.
|
|
|
|
Duplicate cleanup tasks can outlive the tombstone snapshot that
|
|
scheduled them. Holding the shared runtime lock across the final
|
|
re-read, libvirt deletion and marker update prevents an obsolete task
|
|
from deleting a replica created by a same-content reimport.
|
|
"""
|
|
runtime = self.registry.cloud_templates
|
|
with runtime.lock:
|
|
raw = self.store.get(CACHE_TOMBSTONE_KIND, key)
|
|
if raw is None:
|
|
return
|
|
value = self._decode_cloud_template_tombstone(key, raw)
|
|
if (value["deletion_id"] != deletion_id
|
|
or not value["source_deleted"]
|
|
or host_id not in value["pending_hosts"]):
|
|
return
|
|
try:
|
|
service.delete_cloud_template_replicas(
|
|
value["source_volume_name"])
|
|
except Exception as exc:
|
|
from .libvirt_service import VMError
|
|
# Only an accessible pool with no matching volumes is an
|
|
# idempotent success. A missing/misconfigured storage pool is
|
|
# also reported as 404 by libvirt_service and must keep this
|
|
# host pending so an old replica cannot escape reclamation.
|
|
if (not isinstance(exc, VMError)
|
|
or exc.status != 404
|
|
or str(exc) != "云镜像宿主机缓存不存在"):
|
|
raise
|
|
components.store.delete(
|
|
"cloud_template_replica", value["template_id"])
|
|
value["pending_hosts"] = [
|
|
candidate for candidate in value["pending_hosts"]
|
|
if candidate != host_id
|
|
]
|
|
# Retain the terminal record so a response lost after source
|
|
# deletion remains retryable until an intentional reimport.
|
|
self.store.set(
|
|
CACHE_TOMBSTONE_KIND, key,
|
|
json.dumps(value, separators=(",", ":")),
|
|
)
|
|
|
|
async def _cleanup_host_cloud_template_tombstones(
|
|
self, host_id: str) -> None:
|
|
rows = await asyncio.to_thread(
|
|
self.store.list, CACHE_TOMBSTONE_KIND)
|
|
for key, raw in rows.items():
|
|
try:
|
|
tombstone = self._decode_cloud_template_tombstone(key, raw)
|
|
except ValueError:
|
|
LOG.error(
|
|
"Corrupt cloud template cache cleanup record: key=%s", key)
|
|
continue
|
|
if host_id in tombstone["pending_hosts"]:
|
|
await self._cleanup_cloud_template_tombstone_on_host(
|
|
key, tombstone, host_id)
|
|
|
|
def _delete_user_iso_source(
|
|
self, user_id: int, iso_id: str) -> tuple[str, dict]:
|
|
"""Commit one source deletion and its remote-GC tombstone atomically.
|
|
|
|
The shared runtime lock serializes the whole transition. In
|
|
particular, a second DELETE can only observe the first request's
|
|
committed tombstone; it must never remove that durable cleanup record
|
|
after the catalog has disappeared.
|
|
"""
|
|
runtime = self.registry.user_isos
|
|
key = f"{user_id}:{iso_id}"
|
|
with runtime.lock:
|
|
raw = self.store.get("user_iso_tombstone", key)
|
|
if raw is not None:
|
|
tombstone = self._decode_user_iso_tombstone(key, raw)
|
|
if tombstone["source_deleted"]:
|
|
return key, tombstone
|
|
|
|
try:
|
|
record = runtime.get_for(user_id, iso_id)
|
|
except TemplateError as exc:
|
|
if raw is None or exc.status != 404:
|
|
raise
|
|
# The source/catalog removal committed but persisting the
|
|
# final flag failed. Preserve and advance the existing GC
|
|
# record instead of erasing it on a repeated DELETE.
|
|
tombstone["source_deleted"] = True
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
tombstone, separators=(",", ":")))
|
|
if not tombstone["pending_hosts"]:
|
|
self.store.delete("user_iso_tombstone", key)
|
|
return key, tombstone
|
|
host_ids = self._user_iso_replica_hosts(user_id, record)
|
|
tombstone = {
|
|
"user_id": user_id,
|
|
"iso_id": iso_id,
|
|
"volume_name": record["volume_name"],
|
|
"pending_hosts": host_ids,
|
|
"source_deleted": False,
|
|
"created_at": int(time.time()),
|
|
}
|
|
encoded = json.dumps(tombstone, separators=(",", ":"))
|
|
self.store.set("user_iso_tombstone", key, encoded)
|
|
try:
|
|
# Re-enters runtime.lock; it is intentionally an RLock.
|
|
runtime.delete_for(user_id, iso_id)
|
|
except Exception:
|
|
# Only erase the exact provisional record written above. A
|
|
# later committed tombstone must never be removed by a failed
|
|
# or repeated request.
|
|
if self.store.get("user_iso_tombstone", key) == encoded:
|
|
self.store.delete("user_iso_tombstone", key)
|
|
raise
|
|
tombstone["source_deleted"] = True
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
tombstone, separators=(",", ":")))
|
|
if not host_ids:
|
|
self.store.delete("user_iso_tombstone", key)
|
|
return key, tombstone
|
|
|
|
def _user_iso_replica_hosts(self, user_id: int,
|
|
record: dict) -> list[str]:
|
|
replica_key = f"{user_id}:{record['id']}"
|
|
result = []
|
|
host_ids = set(self.store.list("engine_host")) | set(self.registry.components)
|
|
for host_id in host_ids:
|
|
raw = HostStore(self.store, host_id).get(
|
|
"user_iso_replica", replica_key)
|
|
if raw is None:
|
|
continue
|
|
# The exact owner/ISO marker key is already proof that this host
|
|
# may contain the deterministic volume. Reuse requires a strict
|
|
# ready marker, but deletion must fail safe for old or partially
|
|
# damaged marker schemas as well.
|
|
result.append(host_id)
|
|
return sorted(result)
|
|
|
|
@staticmethod
|
|
def _decode_user_iso_tombstone(key: str, raw: str) -> dict:
|
|
from .user_isos import user_iso_volume_name
|
|
try:
|
|
value = json.loads(raw)
|
|
user_id = value["user_id"]
|
|
iso_id = value["iso_id"]
|
|
pending = value["pending_hosts"]
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise ValueError("user ISO tombstone is corrupt") from exc
|
|
if (isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1
|
|
or key != f"{user_id}:{iso_id}"
|
|
or value.get("volume_name") != user_iso_volume_name(iso_id)
|
|
or not isinstance(pending, list)
|
|
or any(not isinstance(host_id, str) or not HOST_ID.fullmatch(host_id)
|
|
for host_id in pending)
|
|
or len(set(pending)) != len(pending)
|
|
or not isinstance(value.get("source_deleted"), bool)):
|
|
raise ValueError("user ISO tombstone is corrupt")
|
|
return value
|
|
|
|
async def _cleanup_user_iso_tombstone_on_host(
|
|
self, key: str, tombstone: dict, host_id: str) -> None:
|
|
if not tombstone.get("source_deleted") or host_id not in tombstone["pending_hosts"]:
|
|
return
|
|
try:
|
|
await self.registry.get(host_id)
|
|
components = self.registry.components[host_id]
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
if profile is None or profile.get("key_locked"):
|
|
return
|
|
service = self._service(profile)
|
|
try:
|
|
await asyncio.to_thread(
|
|
service.delete_user_iso, tombstone["user_id"],
|
|
tombstone["volume_name"])
|
|
except Exception as exc:
|
|
from .libvirt_service import VMError
|
|
if not isinstance(exc, VMError) or exc.status != 404:
|
|
LOG.info(
|
|
"User ISO cache cleanup remains pending: host=%s type=%s",
|
|
host_id, type(exc).__name__)
|
|
return
|
|
await asyncio.to_thread(
|
|
components.store.delete, "user_iso_replica",
|
|
f"{tombstone['user_id']}:{tombstone['iso_id']}")
|
|
await asyncio.to_thread(
|
|
self._complete_user_iso_tombstone_host, key, host_id)
|
|
except Exception as exc:
|
|
LOG.info("User ISO cache cleanup deferred: host=%s type=%s",
|
|
host_id, type(exc).__name__)
|
|
|
|
def _complete_user_iso_tombstone_host(self, key: str, host_id: str) -> None:
|
|
runtime = self.registry.user_isos
|
|
with runtime.lock:
|
|
raw = self.store.get("user_iso_tombstone", key)
|
|
if raw is None:
|
|
return
|
|
value = self._decode_user_iso_tombstone(key, raw)
|
|
value["pending_hosts"] = [
|
|
candidate for candidate in value["pending_hosts"]
|
|
if candidate != host_id
|
|
]
|
|
if value["pending_hosts"]:
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
value, separators=(",", ":")))
|
|
else:
|
|
self.store.delete("user_iso_tombstone", key)
|
|
|
|
async def _cleanup_host_user_iso_tombstones(self, host_id: str) -> None:
|
|
rows = await asyncio.to_thread(self.store.list, "user_iso_tombstone")
|
|
for key, raw in rows.items():
|
|
try:
|
|
tombstone = self._decode_user_iso_tombstone(key, raw)
|
|
except ValueError:
|
|
LOG.error("Corrupt user ISO cache cleanup record: key=%s", key)
|
|
continue
|
|
if host_id in tombstone["pending_hosts"]:
|
|
await self._cleanup_user_iso_tombstone_on_host(
|
|
key, tombstone, host_id)
|
|
|
|
@staticmethod
|
|
def _update_object(body: bytes | None, *, empty: bool = False) -> dict:
|
|
if (body is None or body == b"") and empty:
|
|
return {}
|
|
try:
|
|
value = json.loads(body or b"")
|
|
except (TypeError, ValueError, UnicodeError) as exc:
|
|
raise UpdateError("在线更新参数必须是 JSON 对象", 422) from exc
|
|
if not isinstance(value, dict):
|
|
raise UpdateError("在线更新参数必须是 JSON 对象", 422)
|
|
return value
|
|
|
|
@staticmethod
|
|
def _public_update_job(job: dict) -> dict:
|
|
allowed = {
|
|
"job_id", "status", "phase", "created_at", "started_at", "finished_at",
|
|
"updated", "old_commit", "new_commit", "rollback_ref", "error",
|
|
}
|
|
return {key: value for key, value in job.items() if key in allowed}
|
|
|
|
def _long_running_host_job(self) -> bool:
|
|
cloud_templates = getattr(self.registry, "cloud_templates", None)
|
|
if (self.maintenance_tasks
|
|
or self.registry.user_isos.has_active_jobs()
|
|
or (cloud_templates is not None
|
|
and cloud_templates.has_active_jobs())):
|
|
return True
|
|
for components in self.registry.components.values():
|
|
kinds = ["rescue_import_job", "reinstall_job", "rescue_job"]
|
|
# A child app without the shared runtime can still run the legacy
|
|
# host-scoped importer. Production children use the global runtime;
|
|
# stale pre-upgrade import_job rows must not block updates forever.
|
|
if components.cloud_templates is None:
|
|
kinds.append("import_job")
|
|
for kind in kinds:
|
|
for raw in components.store.list(kind).values():
|
|
try:
|
|
status = json.loads(raw).get("status")
|
|
except (AttributeError, TypeError, ValueError):
|
|
return True
|
|
if status in {"queued", "running", "pending"}:
|
|
return True
|
|
return False
|
|
|
|
async def _update_dispatch(self, method: str, path: str, body: bytes | None,
|
|
delegated: dict, send) -> None:
|
|
if not (delegated.get("is_admin") and delegated.get("is_super_admin")
|
|
and delegated.get("can_manage")):
|
|
raise EngineAuthError("仅超级管理员可以管理在线更新", 403)
|
|
|
|
if path == "/internal/v1/update/settings" and method == "GET":
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
|
|
return
|
|
if path == "/internal/v1/update/settings" and method == "PUT":
|
|
value = self._update_object(body)
|
|
if set(value) - {"access_token", "clear_token"}:
|
|
raise UpdateError("在线更新设置包含未知字段", 422)
|
|
clear = value.get("clear_token", False)
|
|
token = value.get("access_token")
|
|
if not isinstance(clear, bool) or (token is not None and not isinstance(token, str)):
|
|
raise UpdateError("在线更新设置格式无效", 422)
|
|
if clear and token not in {None, ""}:
|
|
raise UpdateError("不能同时保存和删除访问令牌", 422)
|
|
if clear:
|
|
await asyncio.to_thread(self.updater.clear_token)
|
|
elif token not in {None, ""}:
|
|
await asyncio.to_thread(self.updater.set_token, token)
|
|
elif "access_token" not in value:
|
|
raise UpdateError("没有需要保存的在线更新设置", 422)
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
|
|
return
|
|
if path == "/internal/v1/update/check" and method == "POST":
|
|
value = self._update_object(body, empty=True)
|
|
if value:
|
|
raise UpdateError("检查更新不接受额外参数", 422)
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
raise UpdateError("已有更新任务正在执行", 409)
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.check))
|
|
return
|
|
if path == "/internal/v1/update/apply" and method == "POST":
|
|
value = self._update_object(body)
|
|
if set(value) != {"expected_current_commit", "expected_remote_commit"}:
|
|
raise UpdateError("请先检查更新并提交已确认的版本", 422)
|
|
if not all(isinstance(value[key], str) for key in value):
|
|
raise UpdateError("更新版本标识无效", 422)
|
|
async with self._update_lock():
|
|
if self.active_operations or await asyncio.to_thread(self._long_running_host_job):
|
|
raise UpdateError("仍有虚拟机任务正在执行,请等待任务完成后再更新", 409)
|
|
job = await asyncio.to_thread(
|
|
self.updater.create_job,
|
|
expected_current_commit=value["expected_current_commit"],
|
|
expected_remote_commit=value["expected_remote_commit"],
|
|
)
|
|
try:
|
|
await asyncio.to_thread(self.update_trigger)
|
|
job = await self._wait_update_worker_started(job["job_id"])
|
|
except UpdateServiceUncertain:
|
|
raise
|
|
except UpdateError as exc:
|
|
await asyncio.to_thread(
|
|
self.updater.fail_job, job["job_id"],
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
|
|
)
|
|
raise exc
|
|
except Exception as exc:
|
|
await asyncio.to_thread(
|
|
self.updater.fail_job, job["job_id"],
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
|
|
)
|
|
raise UpdateError(
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
|
|
) from exc
|
|
await json_response(send, 202, self._public_update_job(job))
|
|
return
|
|
job_match = re.fullmatch(
|
|
r"/internal/v1/update/jobs/([A-Za-z0-9_-]{16,64})", path)
|
|
if method == "GET" and job_match:
|
|
job = await asyncio.to_thread(self.updater.get_job, job_match[1])
|
|
if job is None:
|
|
raise UpdateError("更新任务不存在", 404)
|
|
await json_response(send, 200, self._public_update_job(job))
|
|
return
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
|
async def _wait_update_worker_started(self, job_id: str) -> dict:
|
|
"""Confirm the one-shot claimed the durable job, not merely its queue."""
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
job = await asyncio.to_thread(self.updater.get_job, job_id)
|
|
if job is None:
|
|
raise UpdateError("独立更新服务未生成任务状态", 503)
|
|
if job.get("status") in {"running", "succeeded"}:
|
|
return job
|
|
if job.get("status") == "failed":
|
|
raise UpdateError("独立更新服务启动后失败,请查看更新服务日志", 503)
|
|
if job.get("status") == "blocked":
|
|
raise UpdateServiceUncertain(
|
|
"更新任务中断且维护锁仍在生效,请通过 SSH 检查更新服务和回滚引用",
|
|
503,
|
|
)
|
|
await asyncio.sleep(0.1)
|
|
# Stop any very-late process before releasing the maintenance gate and
|
|
# marking the job failed. This prevents an unclaimed worker from
|
|
# changing code after ordinary VM mutations have resumed.
|
|
await asyncio.to_thread(self._stop_update_service)
|
|
raise UpdateError("独立更新服务未在限定时间内启动,请查看 systemd 日志", 503)
|
|
|
|
@staticmethod
|
|
def _start_update_service() -> None:
|
|
try:
|
|
result = subprocess.run(
|
|
["/usr/bin/systemctl", "--no-block", "start", "virt-cloud-update.service"],
|
|
cwd="/",
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise UpdateError(
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
|
|
) from exc
|
|
if result.returncode != 0:
|
|
raise UpdateError(
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
|
|
|
|
@staticmethod
|
|
def _update_service_active() -> bool | None:
|
|
try:
|
|
result = subprocess.run(
|
|
["/usr/bin/systemctl", "show", "--property=ActiveState", "--value",
|
|
"virt-cloud-update.service"],
|
|
cwd="/",
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
if result.returncode != 0:
|
|
return None
|
|
state = result.stdout.strip()
|
|
if state in {"active", "activating", "deactivating", "reloading"}:
|
|
return True
|
|
if state in {"inactive", "failed"}:
|
|
return False
|
|
return None
|
|
|
|
@staticmethod
|
|
def _stop_update_service() -> None:
|
|
try:
|
|
result = subprocess.run(
|
|
["/usr/bin/systemctl", "stop", "virt-cloud-update.service"],
|
|
cwd="/",
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise UpdateServiceUncertain(
|
|
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503) from exc
|
|
if result.returncode != 0:
|
|
raise UpdateServiceUncertain(
|
|
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503)
|
|
|
|
async def _reserve_operation(self, host_id, delegated, body, *, resource="vms", method="POST", target=""):
|
|
op_id = delegated["operation_id"]
|
|
# Even rejected signed provisioning requests have a durable outcome.
|
|
# A worker that loses this HTTP response can release its frozen balance
|
|
# only by querying this same operation, without replaying the request.
|
|
from .app import CreateCloudBody, CreateISOBody
|
|
from pydantic import ValidationError
|
|
create = resource == "vms"
|
|
value = {}
|
|
error = None
|
|
try:
|
|
value = json.loads(body) if body else {}
|
|
if not isinstance(value, dict):
|
|
raise ValueError("body must be an object")
|
|
if create:
|
|
(CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value)
|
|
if value.get("guest_ssh_key_id") is not None or value.get("ssh_key_id") is not None:
|
|
error = "公钥必须由 PHP 校验用户归属后传递,不能使用旧公钥 ID"
|
|
except (ValueError, AttributeError, TypeError, ValidationError):
|
|
value = {}
|
|
error = "虚拟机参数校验失败"
|
|
record = {"operation_id": op_id, "host_id": host_id, "user_id": delegated["user_id"],
|
|
"request_hash": hmac.new(self.secret, method.encode() + b"\n" + target.encode() + b"\n" + (body or b""), hashlib.sha256).hexdigest(),
|
|
"vm_uuid": str(uuid.uuid4()) if create else str(uuid.UUID(resource.split("/")[1])),
|
|
"vm_name": value.get("name"), "kind": "create" if create else "mutation",
|
|
"actor_can_manage": bool(delegated.get("can_manage")),
|
|
"actor_is_admin": bool(delegated.get("is_admin")),
|
|
"resource": resource, "method": method, "status": "running", "result": None,
|
|
"safe_to_refund": False, "created_at": int(time.time())}
|
|
if create:
|
|
# New PHP-backed creates always receive the reserved UUID below.
|
|
# Keep this durable marker so cleanup reconciliation checks the
|
|
# UUID-derived resource candidates, while old in-flight records
|
|
# continue to use their historical display-name disk convention.
|
|
record["resource_naming"] = "uuid-v1"
|
|
if resource.endswith("/service-resume"):
|
|
# Store only the allowlisted target-state option. It is needed to
|
|
# replay the exact same convergence request after a lost response;
|
|
# never persist arbitrary request bodies in the operation table.
|
|
record["target_options"] = {
|
|
"start_if_suspended": value.get("start_if_suspended") is True,
|
|
}
|
|
if error:
|
|
# No service, task, VM or volume mutation has been invoked.
|
|
record.update(status="failed", prevalidation_failed=True,
|
|
error=error, safe_to_refund=create, finished_at=int(time.time()))
|
|
fresh = await asyncio.to_thread(self.store.insert, "operation", op_id, json.dumps(record))
|
|
if fresh:
|
|
return record, True
|
|
previous = await self._operation(op_id, delegated, reconcile=False)
|
|
if previous["host_id"] != host_id or previous["user_id"] != delegated["user_id"] or previous["request_hash"] != record["request_hash"]:
|
|
raise EngineAuthError("相同开通操作编号的请求内容不一致", 409)
|
|
return previous, False
|
|
|
|
async def _operation(self, op_id, delegated, *, reconcile=True):
|
|
raw = await asyncio.to_thread(self.store.get, "operation", op_id)
|
|
if raw is None:
|
|
raise EngineAuthError("开通操作不存在", 404)
|
|
record = json.loads(raw)
|
|
if record["user_id"] != delegated["user_id"] and not delegated["is_admin"]:
|
|
raise EngineAuthError("开通操作不存在", 404)
|
|
# Never automatically create again. Recover a known UUID if libvirt
|
|
# committed while the response or process was lost.
|
|
if reconcile and record.get("kind") == "mutation" and record["status"] in {"running", "pending", "uncertain"}:
|
|
return await self._mutation_operation(record)
|
|
if (reconcile and record["status"] in {"running", "uncertain"}
|
|
and op_id not in self.active_operations
|
|
and (record["status"] == "uncertain" or int(time.time()) - record.get("created_at", 0) >= 120)):
|
|
await self.registry.get(record["host_id"])
|
|
components = self.registry.components[record["host_id"]]
|
|
profile = components.manager.current()
|
|
if profile and not profile.get("key_locked"):
|
|
try:
|
|
vm = await asyncio.to_thread(self._service(profile).get_vm, record["vm_uuid"])
|
|
if vm.get("state") != "running" or not vm.get("managed"):
|
|
return record
|
|
from .metrics import host_scope
|
|
await asyncio.to_thread(components.state.users.assign_vm,
|
|
host_scope(profile), record["vm_uuid"], record["user_id"])
|
|
record.update(status="succeeded", result=vm, safe_to_refund=False)
|
|
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
|
|
except Exception:
|
|
# A missing VM can be a transient connection failure or an
|
|
# operation still in progress. It is not permission to retry.
|
|
await self._check_absent(record)
|
|
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
|
|
return record
|
|
|
|
async def _mutation_operation(self, record):
|
|
await self.registry.get(record["host_id"])
|
|
components = self.registry.components[record["host_id"]]
|
|
target_method = self._service_target_method(record.get("resource", ""))
|
|
if target_method is not None:
|
|
if record["operation_id"] in self.active_operations:
|
|
return record
|
|
if (target_method == "service_suspend"
|
|
and not isinstance(record.get("service_was_running_hint"), bool)):
|
|
# Suspending without a durable pre-stop state can permanently
|
|
# lose whether a later paid renewal is allowed to restart the
|
|
# guest. Fail closed instead of replaying a destructive target
|
|
# after the database recovers.
|
|
record.update(
|
|
status="failed",
|
|
error="停机前运行状态未安全保存,已取消服务暂停",
|
|
finished_at=int(time.time()),
|
|
)
|
|
await asyncio.to_thread(
|
|
self.store.set, "operation", record["operation_id"],
|
|
json.dumps(record))
|
|
return record
|
|
profile = components.manager.current()
|
|
if profile is None or profile.get("key_locked"):
|
|
return record
|
|
if (not record.get("actor_can_manage")
|
|
or (not record.get("actor_is_admin")
|
|
and not components.state.users.owns_vm(
|
|
self._host_scope(profile), record["vm_uuid"], record["user_id"]))):
|
|
record.update(status="failed", error="虚拟机不存在", finished_at=int(time.time()))
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
|
|
json.dumps(record))
|
|
return record
|
|
self.active_operations.add(record["operation_id"])
|
|
try:
|
|
async with self._target_lock(record["host_id"], record["vm_uuid"]):
|
|
options = record.get("target_options", {}) \
|
|
if target_method == "service_resume" else {}
|
|
if not isinstance(options, dict) or any(
|
|
key != "start_if_suspended" for key in options):
|
|
options = {}
|
|
result = await asyncio.to_thread(
|
|
getattr(self._service(profile), target_method), record["vm_uuid"],
|
|
**options)
|
|
if (target_method == "service_suspend"
|
|
and isinstance(record.get("service_was_running_hint"), bool)):
|
|
result["service_was_running"] = (
|
|
bool(result.get("service_was_running"))
|
|
or record["service_was_running_hint"])
|
|
except Exception as exc:
|
|
from .libvirt_service import VMError
|
|
if isinstance(exc, VMError) and exc.status < 500:
|
|
record.update(status="failed", error=str(exc), finished_at=int(time.time()))
|
|
else:
|
|
record.update(status="uncertain",
|
|
error="服务状态尚未确认,引擎将安全重试")
|
|
else:
|
|
record.update(status="succeeded", result=result, error=None,
|
|
finished_at=int(time.time()))
|
|
finally:
|
|
self.active_operations.discard(record["operation_id"])
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
|
|
json.dumps(record))
|
|
return record
|
|
kind = "reinstall_job" if record["resource"].endswith("/reinstall") else "rescue_job" if "/rescue" in record["resource"] else None
|
|
if kind is None:
|
|
# A reset/shutdown/delete cannot be safely replayed merely because
|
|
# its response disappeared. Once no executor can still own the
|
|
# operation, terminate it visibly instead of allowing a permanent
|
|
# running/uncertain row to defeat expiry enforcement.
|
|
return await self._finish_orphan_mutation(record)
|
|
scoped = components.store
|
|
job = None
|
|
job_id = record.get("job_id")
|
|
if job_id:
|
|
raw = await asyncio.to_thread(scoped.get, kind, job_id)
|
|
job = json.loads(raw) if raw else None
|
|
else:
|
|
rows = await asyncio.to_thread(scoped.list, kind)
|
|
for candidate_id, raw in rows.items():
|
|
candidate = json.loads(raw)
|
|
if candidate.get("operation_id") == record["operation_id"]:
|
|
job_id, job = candidate_id, candidate
|
|
break
|
|
if job is None:
|
|
return await self._finish_orphan_mutation(record)
|
|
if job.get("owner") != f"php:{record['user_id']}" or job.get("vm_id") != record["vm_uuid"]:
|
|
return record
|
|
record["job_id"] = job_id
|
|
record["result"] = {key: value for key, value in job.items() if key not in {"scope", "owner", "operation_id"}}
|
|
record["result"]["job_id"] = job_id
|
|
status = job.get("status")
|
|
record["status"] = "succeeded" if status in {"completed", "succeeded"} else "running" if status in {"queued", "running"} else "failed"
|
|
if "面板重启" in job.get("phase", "") or "面板在任务执行期间重启" in job.get("error", ""):
|
|
# The asynchronous job registry explicitly confirms that this job
|
|
# is no longer running. Replaying a reinstall/rescue mutation is
|
|
# unsafe, but leaving it uncertain forever would also grant free
|
|
# service by permanently blocking expiry enforcement.
|
|
record["status"] = "failed"
|
|
record["error"] = job.get("error") or "引擎重启导致任务中断,请核查虚拟机状态后重试"
|
|
if job.get("error"):
|
|
record["error"] = job["error"]
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record))
|
|
return record
|
|
|
|
async def _finish_orphan_mutation(self, record):
|
|
if (record["operation_id"] in self.active_operations
|
|
or int(time.time()) - int(record.get("created_at", 0)) < 120):
|
|
return record
|
|
record.update(
|
|
status="failed",
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
finished_at=int(time.time()),
|
|
)
|
|
await asyncio.to_thread(
|
|
self.store.set, "operation", record["operation_id"], json.dumps(record))
|
|
return record
|
|
|
|
@staticmethod
|
|
def _service_target_method(resource: str) -> str | None:
|
|
match = re.fullmatch(
|
|
rf"vms/{VM_ID}/((?:service|network)-(?:suspend|resume))", resource)
|
|
if match is None:
|
|
return None
|
|
return {
|
|
"service-suspend": "service_suspend",
|
|
"service-resume": "service_resume",
|
|
"network-suspend": "network_suspend",
|
|
"network-resume": "network_resume",
|
|
}[match[1]]
|
|
|
|
@staticmethod
|
|
def _host_scope(profile):
|
|
from .metrics import host_scope
|
|
return host_scope(profile)
|
|
|
|
async def _capture_service_was_running_hint(self, host_id, operation, delegated):
|
|
# Run this inside the same per-VM lock as service_suspend. If the
|
|
# engine crashes after stopping QEMU but before saving the response, a
|
|
# replay otherwise sees only a shutoff guest and cannot know whether a
|
|
# later renewal is allowed to start it automatically.
|
|
components = self.registry.components[host_id]
|
|
profile = components.manager.current()
|
|
owns = (profile is not None and not profile.get("key_locked")
|
|
and (delegated.get("is_admin") or components.state.users.owns_vm(
|
|
self._host_scope(profile), operation["vm_uuid"], delegated["user_id"])))
|
|
if not owns:
|
|
return
|
|
before = await asyncio.to_thread(
|
|
self._service(profile).get_vm, operation["vm_uuid"])
|
|
operation["service_was_running_hint"] = before.get("state") == "running"
|
|
# This write is a prerequisite for stopping the guest. Propagate any
|
|
# failure so the child endpoint is never called without a durable hint.
|
|
await asyncio.to_thread(
|
|
self.store.set, "operation", operation["operation_id"],
|
|
json.dumps(operation))
|
|
|
|
async def _check_absent(self, operation):
|
|
operation["safe_to_refund"] = False
|
|
components = self.registry.components.get(operation["host_id"])
|
|
profile = components.manager.current() if components else None
|
|
if profile and not profile.get("key_locked"):
|
|
try:
|
|
absent = await asyncio.to_thread(self._service(profile).provisioning_absent,
|
|
operation["vm_uuid"], operation["vm_name"],
|
|
uuid_resources=operation.get("resource_naming") == "uuid-v1",
|
|
legacy_name_collision=(
|
|
operation.get("resource_naming") is None
|
|
and operation.get("error") == "虚拟机名称已存在"
|
|
))
|
|
if absent:
|
|
await asyncio.to_thread(self._cleanup_absent_records, components,
|
|
profile, operation["vm_uuid"])
|
|
operation.update(status="failed", safe_to_refund=True)
|
|
except Exception:
|
|
pass
|
|
|
|
@staticmethod
|
|
def _cleanup_absent_records(components, profile, vm_id):
|
|
from .metrics import host_scope
|
|
scope = host_scope(profile)
|
|
components.state.users.delete_vm_owner(scope, vm_id)
|
|
components.store.delete("vm_credential", f"{scope}.{vm_id}")
|
|
prefix = scope + "."
|
|
for key, raw in components.store.list("ip_claim").items():
|
|
if key.startswith(prefix) and json.loads(raw).get("vm_uuid") == vm_id:
|
|
components.store.delete("ip_claim", key)
|
|
|
|
def _public_operation(self, record):
|
|
result = {key: record[key] for key in (
|
|
"operation_id", "status", "result", "error", "safe_to_refund",
|
|
"service_was_running_hint", "created_at", "finished_at")
|
|
if key in record}
|
|
# This is deliberately live process state, not a persisted claim. PHP
|
|
# must not close a stale policy job while its original executor can
|
|
# still mutate libvirt. It retries after the executor becomes terminal.
|
|
result["executor_active"] = record.get("operation_id") in self.active_operations
|
|
return result
|
|
|
|
@staticmethod
|
|
def _service(profile):
|
|
from .libvirt_service import VMService
|
|
return VMService(profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"],
|
|
profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""))
|
|
|
|
async def _ticket(self, host_id, vm_id, delegated, send):
|
|
if not delegated.get("session_id"):
|
|
raise EngineAuthError("控制台需要有效的网站会话", 403)
|
|
components = self.registry.components[host_id]
|
|
profile = components.manager.current()
|
|
if profile is None or profile.get("key_locked"):
|
|
raise EngineAuthError("宿主机尚未就绪", 409)
|
|
from .metrics import host_scope
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
|
|
raise EngineAuthError("虚拟机不存在", 404)
|
|
await asyncio.to_thread(self._service(profile).console_port, vm_id)
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
raise EngineAuthError("网站会话已失效", 403)
|
|
ticket = secrets.token_urlsafe(32)
|
|
key = hashlib.sha256(ticket.encode("ascii")).hexdigest()
|
|
record = {"host_id": host_id, "vm_id": vm_id, "actor": delegated, "expires_at": int(time.time()) + 60}
|
|
await asyncio.to_thread(self.store.insert, "console_ticket", key, json.dumps(record))
|
|
await json_response(send, 200, {"ticket": ticket, "expires_in": 60, "websocket_path": "/engine-console"})
|
|
|
|
async def _session_active(self, delegated, host_id, vm_id):
|
|
if self.introspection is not None:
|
|
return await self.introspection(delegated, host_id, vm_id)
|
|
return await asyncio.to_thread(self._introspect, delegated, host_id, vm_id)
|
|
|
|
def _introspect(self, delegated, host_id, vm_id):
|
|
url = self.config.get("php_introspection_url", "")
|
|
parsed = urlsplit(url)
|
|
if (parsed.scheme not in {"http", "https"} or parsed.hostname not in {"127.0.0.1", "::1", "localhost"}
|
|
or parsed.username or parsed.password or parsed.fragment):
|
|
return False
|
|
body = json.dumps({"session_id": delegated["session_id"], "user_id": delegated["user_id"],
|
|
"host_id": host_id, "vm_uuid": vm_id}, separators=(",", ":")).encode()
|
|
secret = decode_secret(self.config.get("introspection_secret", self.config["service_secret"]))
|
|
headers = signed_headers(secret, "POST", parsed.path + ("?" + parsed.query if parsed.query else ""), body, delegated)
|
|
headers["Content-Type"] = "application/json"
|
|
if self.config.get("php_introspection_host"):
|
|
headers["Host"] = self.config["php_introspection_host"]
|
|
try:
|
|
opener = build_opener(ProxyHandler({}), NoRedirects())
|
|
with opener.open(URLRequest(url, body, headers, method="POST"), timeout=5) as response:
|
|
raw = response.read(8193)
|
|
return response.status == 200 and len(raw) <= 8192 and json.loads(raw).get("active") is True
|
|
except Exception:
|
|
return False
|
|
|
|
async def _console(self, scope, receive, send):
|
|
from starlette.websockets import WebSocket
|
|
socket = WebSocket(scope, receive=receive, send=send)
|
|
if scope["path"] != "/engine-console" or scope.get("scheme") != "wss":
|
|
await socket.close(code=4403)
|
|
return
|
|
from .app import _origin_matches_host
|
|
if not _origin_matches_host(socket.headers.get("origin"), socket.headers.get("host")):
|
|
await socket.close(code=4403)
|
|
return
|
|
protocols = [value for value in scope.get("subprotocols", []) if value.startswith("cloud-ticket.")]
|
|
if len(protocols) != 1 or not re.fullmatch(r"cloud-ticket\.[A-Za-z0-9_-]{43}", protocols[0]):
|
|
await socket.close(code=4401)
|
|
return
|
|
key = hashlib.sha256(protocols[0][13:].encode("ascii")).hexdigest()
|
|
raw = await asyncio.to_thread(self.store.get, "console_ticket", key)
|
|
if raw is None:
|
|
await socket.close(code=4401)
|
|
return
|
|
record = json.loads(raw)
|
|
if record["expires_at"] < int(time.time()) or not await asyncio.to_thread(self.store.insert, "ticket_used", key, "1"):
|
|
await socket.close(code=4401)
|
|
return
|
|
await asyncio.to_thread(self.store.delete, "console_ticket", key)
|
|
delegated, host_id, vm_id = record["actor"], record["host_id"], record["vm_id"]
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
await socket.close(code=4403)
|
|
return
|
|
token = TRUSTED_ACTOR.set(delegated)
|
|
relay = watcher = None
|
|
try:
|
|
await self.registry.get(host_id)
|
|
components = self.registry.components[host_id]
|
|
profile = components.manager.current()
|
|
from .metrics import host_scope
|
|
from .console import relay_console
|
|
if profile is None or profile.get("key_locked"):
|
|
await socket.close(code=4409)
|
|
return
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
|
|
await socket.close(code=4403)
|
|
return
|
|
port = await asyncio.to_thread(self._service(profile).console_port, vm_id)
|
|
await socket.accept(subprotocol=protocols[0])
|
|
async def watch():
|
|
while True:
|
|
await asyncio.sleep(self.console_check_interval)
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
return
|
|
current = components.manager.current()
|
|
if current is None or current.get("key_locked"):
|
|
return
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(current), vm_id, delegated["user_id"]):
|
|
return
|
|
relay = asyncio.create_task(relay_console(socket, port=port,
|
|
ssh_target=profile["ssh_target"], ssh_binary="ssh",
|
|
known_hosts_file=profile["ssh_known_hosts_file"], ssh_config_file=profile["ssh_config_file"]))
|
|
watcher = asyncio.create_task(watch())
|
|
await asyncio.wait({relay, watcher}, return_when=asyncio.FIRST_COMPLETED)
|
|
except Exception:
|
|
LOG.warning("Engine console closed after transport/authorization failure")
|
|
finally:
|
|
for task in (relay, watcher):
|
|
if task is not None:
|
|
task.cancel()
|
|
await asyncio.gather(*(task for task in (relay, watcher) if task is not None), return_exceptions=True)
|
|
with suppress(Exception):
|
|
await socket.close(code=4403)
|
|
TRUSTED_ACTOR.reset(token)
|
|
|
|
async def _lifespan(self, receive, send):
|
|
while True:
|
|
message = await receive()
|
|
if message["type"] == "lifespan.startup":
|
|
try:
|
|
await self.registry.start()
|
|
# Remote cache GC must never hold engine readiness hostage:
|
|
# several offline SSH/libvirt hosts can exceed the updater's
|
|
# health-check window. Track cleanup in the background so
|
|
# update apply sees it as maintenance, and skip eager GC
|
|
# while an update job is still completing this restart.
|
|
if not await asyncio.to_thread(self.updater.active_job):
|
|
for host_id in tuple(self.registry.components):
|
|
self._background(
|
|
self._cleanup_host_user_iso_tombstones(host_id))
|
|
self._background(
|
|
self._cleanup_host_cloud_template_tombstones(
|
|
host_id))
|
|
await send({"type": "lifespan.startup.complete"})
|
|
except Exception:
|
|
LOG.exception("Engine startup failed")
|
|
await send({"type": "lifespan.startup.failed", "message": "引擎初始化失败"})
|
|
return
|
|
elif message["type"] == "lifespan.shutdown":
|
|
# A task awaiting asyncio.to_thread cannot stop the underlying
|
|
# libvirt call when cancelled. Drain these bounded cleanup
|
|
# tasks before closing host services so no hidden mutation can
|
|
# outlive the engine lifespan.
|
|
await asyncio.gather(
|
|
*tuple(self.maintenance_tasks), return_exceptions=True)
|
|
await self.registry.close()
|
|
await send({"type": "lifespan.shutdown.complete"})
|
|
return
|
|
|
|
|
|
def create_engine():
|
|
default = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
|
|
config = private_config(Path(os.environ.get("CLOUD_ENGINE_CONFIG", str(default))))
|
|
secret = decode_secret(config["service_secret"])
|
|
del secret
|
|
key = master_key(Path(config["master_key_file"]))
|
|
store = MainStore(config["main_database"], key)
|
|
logs = LogStore(config["log_database"], key)
|
|
return EngineApplication(config, store, logs)
|