|
|
"""Private PHP -> libvirt boundary. The legacy browser app is never exposed.
|
|
|
|
|
|
Only allowlisted VM/host/template resources are dispatched to per-host services.
|
|
|
The public console endpoint accepts a short-lived, one-use ticket instead of a
|
|
|
PHP cookie, and rechecks the PHP session while the console remains connected.
|
|
|
"""
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import asyncio
|
|
|
import errno
|
|
|
import hashlib
|
|
|
import hmac
|
|
|
import inspect
|
|
|
import json
|
|
|
import logging
|
|
|
import os
|
|
|
import re
|
|
|
import secrets
|
|
|
import stat
|
|
|
import subprocess
|
|
|
import tempfile
|
|
|
import threading
|
|
|
import time
|
|
|
import uuid
|
|
|
from contextlib import AsyncExitStack, suppress
|
|
|
from pathlib import Path
|
|
|
from urllib.parse import parse_qsl, urlsplit
|
|
|
from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLRequest,
|
|
|
build_opener)
|
|
|
|
|
|
|
|
|
from .cloud_templates import (
|
|
|
MAX_DOWNLOAD_SECONDS,
|
|
|
MAX_IMAGE_BYTES,
|
|
|
TemplateError,
|
|
|
template_volume_name,
|
|
|
validate_template_metadata,
|
|
|
validate_template_name,
|
|
|
verify_qcow2_image,
|
|
|
)
|
|
|
from .cloud_template_runtime import (
|
|
|
CACHE_TOMBSTONE_KIND,
|
|
|
MIN_DOWNLOAD_BYTES,
|
|
|
CloudTemplateRuntime,
|
|
|
)
|
|
|
from .engine_context import (EngineComponents, HostStore, LogStore, MainStore,
|
|
|
TRUSTED_ACTOR, actor_session)
|
|
|
from .engine_security import (EngineAuthError, authenticate, decode_secret,
|
|
|
signed_headers)
|
|
|
from .updater import UpdateError, Updater
|
|
|
from .user_isos import UserISORuntime
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
|
OP_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z")
|
|
|
VM_ID = r"[0-9a-fA-F-]{36}"
|
|
|
JSON_LIMIT = 1024 * 1024
|
|
|
CLOUD_UPLOAD_IDLE_SECONDS = 60
|
|
|
CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024
|
|
|
CACHE_CLEANUP_RETRY_SECONDS = 30.0
|
|
|
DELETE_CLEANUP_SNAPSHOT_LIMIT = 64
|
|
|
SNAPSHOT_PUBLIC_ERROR_CODES = {
|
|
|
"snapshot_guest_agent_unavailable",
|
|
|
"live_snapshot_unsupported",
|
|
|
"snapshot_pool_unsupported",
|
|
|
"snapshot_result_uncertain",
|
|
|
}
|
|
|
FIREWALL_PUBLIC_ERROR_CODES = {
|
|
|
"firewall_capability_missing",
|
|
|
"firewall_rule_invalid",
|
|
|
"firewall_apply_failed",
|
|
|
"firewall_result_uncertain",
|
|
|
"firewall_drift_detected",
|
|
|
"firewall_cleanup_pending",
|
|
|
}
|
|
|
REINSTALL_PUBLIC_ERROR_CODES = {
|
|
|
"reinstall_cleanup_pending",
|
|
|
"reinstall_result_uncertain",
|
|
|
}
|
|
|
# Compatibility with reinstall rows written before the explicit commit marker
|
|
|
# was introduced. These phases are only emitted after the replacement domain
|
|
|
# has been defined/started or while its post-commit result is being saved.
|
|
|
REINSTALL_COMMITTED_PHASES = frozenset({
|
|
|
"cleaning_up", "cleanup_pending", "result_uncertain", "completed",
|
|
|
"重装已完成,但凭据或任务记录保存失败",
|
|
|
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
|
|
|
})
|
|
|
DELETE_PUBLIC_ERROR_CODES = {
|
|
|
"public_ip_claim_cleanup_pending",
|
|
|
"vm_delete_snapshot_cleanup_pending",
|
|
|
"vm_delete_firewall_cleanup_pending",
|
|
|
"vm_delete_volume_cleanup_pending",
|
|
|
}
|
|
|
POWER_ACTIONS = frozenset({"start", "shutdown", "reboot", "poweroff", "reset"})
|
|
|
POWER_ACTION_EDGE_ACTIONS = frozenset({"reboot", "reset"})
|
|
|
POWER_ACTION_TERMINAL_STATES = {
|
|
|
"start": frozenset({"running"}),
|
|
|
"shutdown": frozenset({"shutoff", "stopped"}),
|
|
|
"poweroff": frozenset({"shutoff", "stopped"}),
|
|
|
}
|
|
|
|
|
|
|
|
|
def _reinstall_committed(job: dict) -> bool:
|
|
|
if job.get("reinstall_committed") is True:
|
|
|
return True
|
|
|
if job.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES:
|
|
|
return True
|
|
|
phase = job.get("phase")
|
|
|
if isinstance(phase, str) and phase.strip() in REINSTALL_COMMITTED_PHASES:
|
|
|
return True
|
|
|
if job.get("cleanup_pending") is True:
|
|
|
return True
|
|
|
warning = job.get("warning")
|
|
|
return isinstance(warning, str) and bool(warning.strip())
|
|
|
|
|
|
|
|
|
def _reinstall_cleanup_pending(job: dict) -> bool:
|
|
|
phase = job.get("phase")
|
|
|
if isinstance(phase, str):
|
|
|
phase = phase.strip()
|
|
|
if job.get("error_code") == "reinstall_cleanup_pending":
|
|
|
return True
|
|
|
if job.get("cleanup_pending") is True:
|
|
|
return True
|
|
|
if phase in {"cleanup_pending", "重装完成,但旧系统盘清理待核对"}:
|
|
|
return True
|
|
|
warning = job.get("warning")
|
|
|
return isinstance(warning, str) and warning.strip().startswith("旧存储卷")
|
|
|
|
|
|
|
|
|
READ_PATHS = (
|
|
|
r"host", r"host/network-pools", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms",
|
|
|
r"vm-states", r"vm-metadata",
|
|
|
r"user-isos/[0-9a-f]{32}",
|
|
|
rf"vms/{VM_ID}(?:/state|/metrics|/traffic/daily|/rescue|"
|
|
|
r"/firewall|/snapshots(?:/[0-9a-f]{32})?)?",
|
|
|
r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+",
|
|
|
)
|
|
|
WRITE_PATHS = {
|
|
|
"POST": (r"host/(?:scan|unlock|test)",
|
|
|
r"vm-templates/(?:import|upload)", r"rescue-templates",
|
|
|
r"user-isos/import",
|
|
|
r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|network|settings|rescue|console-ticket)",
|
|
|
rf"vms/{VM_ID}/snapshots(?:/[0-9a-f]{{32}}/restore)?",
|
|
|
rf"vms/{VM_ID}/(?:service|network)-(?:suspend|resume)",
|
|
|
rf"vms/{VM_ID}/rescue/(?:exit|force-start)", r"reinstall-jobs/[A-Za-z0-9_-]+/ack-password"),
|
|
|
"PUT": (r"host", r"host/network-pools/[0-9a-f]{32}",
|
|
|
rf"vms/{VM_ID}/firewall"),
|
|
|
"PATCH": (r"vm-templates/[A-Za-z0-9_-]+",),
|
|
|
"DELETE": (rf"vms/{VM_ID}(?:/snapshots/[0-9a-f]{{32}})?",
|
|
|
r"vm-templates/[A-Za-z0-9_-]+",
|
|
|
r"rescue-templates/[A-Za-z0-9_-]+",
|
|
|
r"user-isos/[0-9a-f]{32}"),
|
|
|
}
|
|
|
|
|
|
|
|
|
class _CloudUploadWriter:
|
|
|
"""Own a temporary upload descriptor used only from worker threads."""
|
|
|
|
|
|
def __init__(self, directory: Path) -> None:
|
|
|
descriptor, raw_path = tempfile.mkstemp(
|
|
|
prefix=".cloud-image-upload-",
|
|
|
suffix=".qcow2",
|
|
|
dir=directory,
|
|
|
)
|
|
|
try:
|
|
|
os.fchmod(descriptor, 0o600)
|
|
|
except BaseException:
|
|
|
os.close(descriptor)
|
|
|
Path(raw_path).unlink(missing_ok=True)
|
|
|
raise
|
|
|
self.path = Path(raw_path)
|
|
|
self._descriptor = descriptor
|
|
|
self._lock = threading.Lock()
|
|
|
|
|
|
def write(self, chunk: memoryview) -> None:
|
|
|
"""Write the complete bounded chunk before accepting another one."""
|
|
|
with self._lock:
|
|
|
if self._descriptor < 0:
|
|
|
raise OSError(errno.EBADF, "cloud upload is closed")
|
|
|
remaining = chunk
|
|
|
while remaining:
|
|
|
written = os.write(self._descriptor, remaining)
|
|
|
if written <= 0:
|
|
|
raise OSError(errno.EIO, "cloud upload write made no progress")
|
|
|
remaining = remaining[written:]
|
|
|
|
|
|
def sync_and_close(self) -> None:
|
|
|
"""Durably flush and close the descriptor before image validation."""
|
|
|
with self._lock:
|
|
|
if self._descriptor < 0:
|
|
|
return
|
|
|
descriptor = self._descriptor
|
|
|
try:
|
|
|
os.fsync(descriptor)
|
|
|
finally:
|
|
|
try:
|
|
|
os.close(descriptor)
|
|
|
finally:
|
|
|
self._descriptor = -1
|
|
|
|
|
|
def close(self) -> None:
|
|
|
"""Close after an interrupted request, waiting for an in-flight write."""
|
|
|
with self._lock:
|
|
|
if self._descriptor < 0:
|
|
|
return
|
|
|
descriptor = self._descriptor
|
|
|
try:
|
|
|
os.close(descriptor)
|
|
|
finally:
|
|
|
self._descriptor = -1
|
|
|
|
|
|
|
|
|
class NoRedirects(HTTPRedirectHandler):
|
|
|
def redirect_request(self, *args, **kwargs):
|
|
|
raise ValueError("Session introspection redirects are not allowed")
|
|
|
|
|
|
|
|
|
class UpdateServiceUncertain(UpdateError):
|
|
|
"""The worker could not be confirmed stopped, so its job must stay active."""
|
|
|
|
|
|
|
|
|
def private_config(path: Path) -> dict:
|
|
|
info = path.lstat()
|
|
|
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 or info.st_uid != os.geteuid():
|
|
|
raise ValueError("引擎配置必须为运行用户拥有的普通 0600 文件")
|
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
|
try:
|
|
|
raw = os.read(descriptor, JSON_LIMIT + 1)
|
|
|
finally:
|
|
|
os.close(descriptor)
|
|
|
if len(raw) > JSON_LIMIT:
|
|
|
raise ValueError("引擎配置过大")
|
|
|
value = json.loads(raw)
|
|
|
if not isinstance(value, dict):
|
|
|
raise ValueError("引擎配置无效")
|
|
|
return value
|
|
|
|
|
|
|
|
|
def master_key(path: Path) -> bytes:
|
|
|
from .database_bootstrap import DatabaseBootstrap
|
|
|
if not path.is_absolute():
|
|
|
raise ValueError("主密钥路径必须是绝对路径")
|
|
|
bootstrap = DatabaseBootstrap(path.parent)
|
|
|
bootstrap.key_file = path
|
|
|
return bootstrap.master_key(create=True)
|
|
|
|
|
|
|
|
|
class HostRegistry:
|
|
|
def __init__(self, config, store, log_store, *, factory=None, user_isos=None,
|
|
|
cloud_templates=None):
|
|
|
self.config = config
|
|
|
self.store = store
|
|
|
self.log_store = log_store
|
|
|
self.factory = factory
|
|
|
self.user_isos = user_isos or UserISORuntime(
|
|
|
store, Path(config["data_dir"]))
|
|
|
self._owns_user_isos = user_isos is None
|
|
|
self.cloud_templates = cloud_templates or CloudTemplateRuntime(
|
|
|
store, Path(config["data_dir"]))
|
|
|
self._owns_cloud_templates = cloud_templates is None
|
|
|
self.components: dict[str, EngineComponents] = {}
|
|
|
self.apps = {}
|
|
|
# Constructed by synchronous factories before an event loop may exist.
|
|
|
# Bind the lock lazily inside the ASGI lifespan/request loop.
|
|
|
self.lock: asyncio.Lock | None = None
|
|
|
self.lifespans = AsyncExitStack()
|
|
|
|
|
|
async def start(self):
|
|
|
if self.lock is None:
|
|
|
self.lock = asyncio.Lock()
|
|
|
await asyncio.to_thread(self.user_isos.restore_jobs)
|
|
|
await asyncio.to_thread(self.cloud_templates.restore_jobs)
|
|
|
await self.lifespans.__aenter__()
|
|
|
for host_id in await asyncio.to_thread(self.store.list, "engine_host"):
|
|
|
await self.get(host_id)
|
|
|
|
|
|
async def close(self):
|
|
|
await self.lifespans.aclose()
|
|
|
if self._owns_user_isos:
|
|
|
await asyncio.to_thread(self.user_isos.close)
|
|
|
if self._owns_cloud_templates:
|
|
|
await asyncio.to_thread(self.cloud_templates.close)
|
|
|
|
|
|
async def get(self, host_id: str):
|
|
|
if not HOST_ID.fullmatch(host_id):
|
|
|
raise EngineAuthError("宿主机标识无效", 400)
|
|
|
if self.lock is None:
|
|
|
self.lock = asyncio.Lock()
|
|
|
async with self.lock:
|
|
|
if host_id in self.apps:
|
|
|
return self.apps[host_id]
|
|
|
from .app import create_app
|
|
|
from .config import Settings
|
|
|
directory = Path(self.config["data_dir"]) / "hosts" / host_id
|
|
|
if self.factory is not None:
|
|
|
try:
|
|
|
inspect.signature(self.factory).bind(
|
|
|
host_id, self.user_isos, self.cloud_templates)
|
|
|
except TypeError:
|
|
|
try:
|
|
|
inspect.signature(self.factory).bind(host_id, self.user_isos)
|
|
|
except TypeError:
|
|
|
components, app = self.factory(host_id)
|
|
|
else:
|
|
|
components, app = self.factory(host_id, self.user_isos)
|
|
|
else:
|
|
|
components, app = self.factory(
|
|
|
host_id, self.user_isos, self.cloud_templates)
|
|
|
else:
|
|
|
components = await asyncio.to_thread(EngineComponents.for_host,
|
|
|
self.store, self.log_store, host_id, directory,
|
|
|
self.user_isos, self.cloud_templates)
|
|
|
settings = Settings(data_dir=directory)
|
|
|
app = await asyncio.to_thread(create_app, settings, engine_components=components)
|
|
|
self.components[host_id] = components
|
|
|
await self.lifespans.enter_async_context(app.router.lifespan_context(app))
|
|
|
self.apps[host_id] = app
|
|
|
return app
|
|
|
|
|
|
|
|
|
async def json_response(send, status: int, payload):
|
|
|
body = json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode()
|
|
|
await send({"type": "http.response.start", "status": status,
|
|
|
"headers": [(b"content-type", b"application/json"), (b"cache-control", b"no-store")]})
|
|
|
await send({"type": "http.response.body", "body": body})
|
|
|
|
|
|
|
|
|
def raw_target(scope) -> str:
|
|
|
path = scope.get("raw_path", scope["path"].encode("ascii")).decode("ascii")
|
|
|
query = scope.get("query_string", b"")
|
|
|
return path + ("?" + query.decode("ascii") if query else "")
|
|
|
|
|
|
|
|
|
def request_headers(scope):
|
|
|
values = {}
|
|
|
for key, value in scope.get("headers", []):
|
|
|
name = key.decode("ascii").lower()
|
|
|
if name.startswith("x-cloud-") and name in values:
|
|
|
raise EngineAuthError("重复的内部认证头")
|
|
|
values[name] = value.decode("latin-1")
|
|
|
return values
|
|
|
|
|
|
|
|
|
class EngineApplication:
|
|
|
def __init__(self, config, store, log_store, *, registry=None, introspection=None,
|
|
|
updater=None, update_trigger=None):
|
|
|
self.config = config
|
|
|
self.store = store
|
|
|
self.secret = decode_secret(config["service_secret"])
|
|
|
self.registry = registry or HostRegistry(config, store, log_store)
|
|
|
self.introspection = introspection
|
|
|
project_dir = Path(config.get("project_dir", Path(__file__).resolve().parents[2]))
|
|
|
self.updater = updater or Updater(
|
|
|
project_dir, store, worker_active=self._update_service_active)
|
|
|
self.update_trigger = update_trigger or self._start_update_service
|
|
|
# Serializes the transition into update maintenance mode with every
|
|
|
# request that can mutate libvirt or durable operation state. The
|
|
|
# lock is created lazily so synchronous test/application factories do
|
|
|
# not bind it to an event loop that has not started yet.
|
|
|
self.update_coordination_lock: asyncio.Lock | None = None
|
|
|
self.active_operations: set[str] = set()
|
|
|
self.maintenance_tasks: set[asyncio.Task] = set()
|
|
|
self.cache_cleanup_event: asyncio.Event | None = None
|
|
|
self.cache_cleanup_supervisor: asyncio.Task | None = None
|
|
|
self.cache_cleanup_retry_seconds = CACHE_CLEANUP_RETRY_SECONDS
|
|
|
# Target-state operations are idempotent individually, but two
|
|
|
# different operation IDs can request opposite states at the same
|
|
|
# time (for example traffic suspension ending at a period boundary).
|
|
|
# Serialize those mutations per VM so the later request always wins
|
|
|
# on the hypervisor as well as in the billing database.
|
|
|
self.target_locks: dict[tuple[str, str], asyncio.Lock] = {}
|
|
|
self.console_check_interval = 15
|
|
|
|
|
|
def _background(self, coroutine) -> asyncio.Task:
|
|
|
task = asyncio.create_task(coroutine)
|
|
|
self.maintenance_tasks.add(task)
|
|
|
task.add_done_callback(self.maintenance_tasks.discard)
|
|
|
return task
|
|
|
|
|
|
def _wake_cache_cleanup(self) -> bool:
|
|
|
if self.cache_cleanup_event is None:
|
|
|
return False
|
|
|
self.cache_cleanup_event.set()
|
|
|
return True
|
|
|
|
|
|
def _pending_cache_cleanup_hosts(self) -> dict[str, set[str]]:
|
|
|
"""Return only hosts named by a durable, committed delete event."""
|
|
|
pending: dict[str, set[str]] = {}
|
|
|
for key, raw in self.store.list("user_iso_tombstone").items():
|
|
|
try:
|
|
|
tombstone = self._decode_user_iso_tombstone(key, raw)
|
|
|
except ValueError:
|
|
|
LOG.error("Corrupt user ISO cache cleanup record: key=%s", key)
|
|
|
continue
|
|
|
if tombstone.get("source_deleted"):
|
|
|
for host_id in tombstone["pending_hosts"]:
|
|
|
pending.setdefault(host_id, set()).add("user_iso")
|
|
|
for key, raw in self.store.list(CACHE_TOMBSTONE_KIND).items():
|
|
|
try:
|
|
|
tombstone = self._decode_cloud_template_tombstone(key, raw)
|
|
|
except ValueError:
|
|
|
LOG.error(
|
|
|
"Corrupt cloud template cache cleanup record: key=%s", key)
|
|
|
continue
|
|
|
if tombstone.get("source_deleted"):
|
|
|
for host_id in tombstone["pending_hosts"]:
|
|
|
pending.setdefault(host_id, set()).add("cloud_template")
|
|
|
return pending
|
|
|
|
|
|
async def _launch_cache_cleanup_pass(
|
|
|
self, pending: dict[str, set[str]]) -> list[asyncio.Task]:
|
|
|
# Coordinate only the transition into remote maintenance. The actual
|
|
|
# SSH/libvirt work is tracked in maintenance_tasks, so update apply is
|
|
|
# rejected while a deletion pass is mutating any host.
|
|
|
async with self._update_lock():
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
return []
|
|
|
tasks = []
|
|
|
for host_id, kinds in pending.items():
|
|
|
if "user_iso" in kinds:
|
|
|
tasks.append(self._background(
|
|
|
self._cleanup_host_user_iso_tombstones(host_id)))
|
|
|
if "cloud_template" in kinds:
|
|
|
tasks.append(self._background(
|
|
|
self._cleanup_host_cloud_template_tombstones(host_id)))
|
|
|
return tasks
|
|
|
|
|
|
async def _wait_cache_cleanup_retry(self) -> None:
|
|
|
event = self.cache_cleanup_event
|
|
|
if event is None:
|
|
|
return
|
|
|
try:
|
|
|
await asyncio.wait_for(
|
|
|
event.wait(), timeout=max(0.01, self.cache_cleanup_retry_seconds))
|
|
|
except asyncio.TimeoutError:
|
|
|
pass
|
|
|
event.clear()
|
|
|
|
|
|
async def _cache_cleanup_loop(self) -> None:
|
|
|
"""Retry durable delete tombstones without coupling them to reads."""
|
|
|
event = self.cache_cleanup_event
|
|
|
if event is None:
|
|
|
return
|
|
|
while True:
|
|
|
await event.wait()
|
|
|
event.clear()
|
|
|
while True:
|
|
|
try:
|
|
|
pending = await asyncio.to_thread(
|
|
|
self._pending_cache_cleanup_hosts)
|
|
|
if not pending:
|
|
|
break
|
|
|
tasks = await self._launch_cache_cleanup_pass(pending)
|
|
|
if tasks:
|
|
|
# asyncio.wait never cancels the tracked maintenance
|
|
|
# tasks if this supervisor is cancelled at shutdown.
|
|
|
await asyncio.wait(tasks)
|
|
|
remaining = await asyncio.to_thread(
|
|
|
self._pending_cache_cleanup_hosts)
|
|
|
if not remaining:
|
|
|
break
|
|
|
except asyncio.CancelledError:
|
|
|
raise
|
|
|
except Exception as exc:
|
|
|
LOG.warning(
|
|
|
"Delete cache cleanup supervisor deferred: type=%s",
|
|
|
type(exc).__name__)
|
|
|
await self._wait_cache_cleanup_retry()
|
|
|
|
|
|
def _target_lock(self, host_id: str, vm_id: str) -> asyncio.Lock:
|
|
|
key = (host_id, vm_id)
|
|
|
lock = self.target_locks.get(key)
|
|
|
if lock is None:
|
|
|
lock = asyncio.Lock()
|
|
|
self.target_locks[key] = lock
|
|
|
return lock
|
|
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
|
if scope["type"] == "lifespan":
|
|
|
await self._lifespan(receive, send)
|
|
|
return
|
|
|
if scope["type"] == "websocket":
|
|
|
try:
|
|
|
await self._console(scope, receive, send)
|
|
|
except Exception:
|
|
|
LOG.warning("Engine console rejected because a dependency failed")
|
|
|
await send({"type": "websocket.close", "code": 4403})
|
|
|
return
|
|
|
if scope["type"] != "http":
|
|
|
return
|
|
|
token = None
|
|
|
response_started = False
|
|
|
async def tracked_send(message):
|
|
|
nonlocal response_started
|
|
|
if message["type"] == "http.response.start":
|
|
|
response_started = True
|
|
|
await send(message)
|
|
|
try:
|
|
|
# Public reverse proxies must never forward these routes. Reject
|
|
|
# non-loopback peers even when they have spoofed forwarded headers.
|
|
|
client = scope.get("client")
|
|
|
if client and client[0] not in {"127.0.0.1", "::1", "testclient"}:
|
|
|
raise EngineAuthError("内部引擎仅接受本机服务请求", 403)
|
|
|
headers = request_headers(scope)
|
|
|
target = raw_target(scope)
|
|
|
upload_resource = None
|
|
|
if scope["method"] == "POST":
|
|
|
if scope["path"] == "/internal/v1/vm-templates/upload":
|
|
|
upload_resource = "vm-templates/upload"
|
|
|
elif re.fullmatch(
|
|
|
r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates",
|
|
|
scope["path"]):
|
|
|
upload_resource = "rescue-templates"
|
|
|
upload = upload_resource is not None
|
|
|
if upload:
|
|
|
digest = headers.get("x-cloud-content-sha256", "")
|
|
|
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
|
|
|
scope["method"], target, digest, self.store)
|
|
|
self._authorize_route(
|
|
|
scope["method"], upload_resource, delegated)
|
|
|
checksum = hashlib.sha256()
|
|
|
validation = {"complete": False, "valid": False}
|
|
|
scope = {
|
|
|
**scope,
|
|
|
"engine_upload_validation": validation,
|
|
|
"engine_upload_digest": digest,
|
|
|
}
|
|
|
async def verified_receive():
|
|
|
message = await receive()
|
|
|
if message["type"] == "http.request":
|
|
|
chunk = message.get("body", b"")
|
|
|
checksum.update(chunk)
|
|
|
if not message.get("more_body", False):
|
|
|
validation.update(complete=True,
|
|
|
valid=hmac.compare_digest(checksum.hexdigest(), digest))
|
|
|
return message
|
|
|
child_receive = verified_receive
|
|
|
body = None
|
|
|
else:
|
|
|
chunks = []
|
|
|
size = 0
|
|
|
while True:
|
|
|
message = await receive()
|
|
|
if message["type"] == "http.disconnect":
|
|
|
return
|
|
|
chunk = message.get("body", b"")
|
|
|
size += len(chunk)
|
|
|
if size > JSON_LIMIT:
|
|
|
raise EngineAuthError("内部请求超过 1 MiB", 413)
|
|
|
chunks.append(chunk)
|
|
|
if not message.get("more_body", False):
|
|
|
break
|
|
|
body = b"".join(chunks)
|
|
|
delegated = await asyncio.to_thread(authenticate, headers, self.secret,
|
|
|
scope["method"], target, hashlib.sha256(body).hexdigest(), self.store)
|
|
|
delivered = False
|
|
|
async def replay_receive():
|
|
|
nonlocal delivered
|
|
|
if not delivered:
|
|
|
delivered = True
|
|
|
return {"type": "http.request", "body": body, "more_body": False}
|
|
|
return await receive()
|
|
|
child_receive = replay_receive
|
|
|
token = TRUSTED_ACTOR.set(delegated)
|
|
|
await self._dispatch(scope, child_receive, tracked_send, body, delegated)
|
|
|
except asyncio.CancelledError:
|
|
|
raise
|
|
|
except EngineAuthError as exc:
|
|
|
if not response_started:
|
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
except UpdateError as exc:
|
|
|
if not response_started:
|
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
except Exception as exc:
|
|
|
LOG.exception("Private engine request failed")
|
|
|
if not response_started:
|
|
|
await json_response(send, 503, {"error": "引擎服务暂时不可用,请检查引擎日志"})
|
|
|
finally:
|
|
|
if token is not None:
|
|
|
TRUSTED_ACTOR.reset(token)
|
|
|
|
|
|
@staticmethod
|
|
|
def _authorize_route(method: str, resource: str, delegated: dict):
|
|
|
paths = READ_PATHS if method == "GET" else WRITE_PATHS.get(method, ())
|
|
|
if not any(re.fullmatch(pattern, resource) for pattern in paths):
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
administration = (resource.startswith("host") or resource.startswith("vm-templates/")
|
|
|
or resource.startswith("rescue-templates/"))
|
|
|
administration_write = method != "GET" and (administration or resource == "rescue-templates")
|
|
|
if administration and not delegated["is_admin"]:
|
|
|
raise EngineAuthError("仅管理员可访问此资源", 403)
|
|
|
if method != "GET" and not delegated["can_manage"]:
|
|
|
raise EngineAuthError("当前账户不允许操作", 403)
|
|
|
if administration_write and not delegated["is_super_admin"]:
|
|
|
raise EngineAuthError("仅超级管理员可修改宿主机和模版", 403)
|
|
|
|
|
|
async def _dispatch(self, scope, receive, send, body, delegated):
|
|
|
path = scope["path"]
|
|
|
if path.startswith("/internal/v1/update/"):
|
|
|
await self._update_dispatch(scope["method"], path, body, delegated, send)
|
|
|
return
|
|
|
method = scope["method"]
|
|
|
operation_reconcile = (
|
|
|
method == "GET"
|
|
|
and re.fullmatch(r"/internal/v1/operations/[A-Za-z0-9_-]{8,80}", path) is not None
|
|
|
and scope.get("query_string", b"") != b"reconcile=false"
|
|
|
)
|
|
|
firewall_reconcile = False
|
|
|
if operation_reconcile:
|
|
|
operation_id = path.rsplit("/", 1)[-1]
|
|
|
raw_operation = await asyncio.to_thread(
|
|
|
self.store.get, "operation", operation_id)
|
|
|
try:
|
|
|
stored_operation = json.loads(raw_operation)
|
|
|
except (TypeError, ValueError):
|
|
|
stored_operation = None
|
|
|
firewall_reconcile = (
|
|
|
isinstance(stored_operation, dict)
|
|
|
and self._is_firewall_resource(
|
|
|
stored_operation.get("resource", ""))
|
|
|
)
|
|
|
# A browser upload can spend minutes waiting for request-body chunks.
|
|
|
# Its handler acquires the update coordination lock only to reserve and
|
|
|
# finally queue the import, so unrelated VM writes remain available.
|
|
|
if method == "POST" and path == "/internal/v1/vm-templates/upload":
|
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
|
return
|
|
|
firewall_put = (
|
|
|
method == "PUT"
|
|
|
and re.fullmatch(
|
|
|
rf"/internal/v1/hosts/[A-Za-z0-9][A-Za-z0-9_-]{{0,63}}/"
|
|
|
rf"vms/{VM_ID}/firewall",
|
|
|
path,
|
|
|
) is not None
|
|
|
)
|
|
|
if firewall_put or firewall_reconcile:
|
|
|
# Firewall mutations use the per-VM target lock below. They do not
|
|
|
# hold the engine-wide update coordination lock while waiting on a
|
|
|
# remote libvirt host, so other VMs and read-only VM reads remain
|
|
|
# concurrent. A temporary marker closes the race with an update
|
|
|
# starting between the maintenance check and durable operation
|
|
|
# reservation, without serializing the remote host call. If an
|
|
|
# update already owns the coordination lock, reject before placing
|
|
|
# that marker; there is no await between this check and the add.
|
|
|
if (self.update_coordination_lock is not None
|
|
|
and self.update_coordination_lock.locked()):
|
|
|
raise EngineAuthError(
|
|
|
"面板正在更新,暂时不能核对或修改防火墙", 409)
|
|
|
maintenance_gate = "firewall-gate:" + secrets.token_hex(16)
|
|
|
self.active_operations.add(maintenance_gate)
|
|
|
try:
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
raise EngineAuthError(
|
|
|
"面板正在更新,暂时不能核对或修改防火墙", 409)
|
|
|
await self._dispatch_regular(
|
|
|
scope, receive, send, body, delegated)
|
|
|
finally:
|
|
|
self.active_operations.discard(maintenance_gate)
|
|
|
return
|
|
|
if method != "GET" or operation_reconcile:
|
|
|
async with self._update_lock():
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
|
return
|
|
|
await self._dispatch_regular(scope, receive, send, body, delegated)
|
|
|
|
|
|
def _update_lock(self) -> asyncio.Lock:
|
|
|
if self.update_coordination_lock is None:
|
|
|
self.update_coordination_lock = asyncio.Lock()
|
|
|
return self.update_coordination_lock
|
|
|
|
|
|
async def _dispatch_regular(self, scope, receive, send, body, delegated):
|
|
|
path = scope["path"]
|
|
|
if scope["method"] == "POST" and path == "/internal/v1/hosts/scan":
|
|
|
if not delegated["is_super_admin"] or not delegated["can_manage"]:
|
|
|
raise EngineAuthError("仅超级管理员可以扫描宿主机", 403)
|
|
|
from .app import HostScanBody
|
|
|
from .remote_host import RemoteHostManager
|
|
|
from pydantic import ValidationError
|
|
|
try:
|
|
|
value = HostScanBody.model_validate_json(body)
|
|
|
except ValidationError as exc:
|
|
|
raise EngineAuthError("宿主机扫描参数无效", 422) from exc
|
|
|
manager = await asyncio.to_thread(RemoteHostManager,
|
|
|
Path(self.config["data_dir"]) / "discovery")
|
|
|
try:
|
|
|
result = await asyncio.to_thread(manager.scan, value.host, value.port)
|
|
|
await json_response(send, 200, result)
|
|
|
finally:
|
|
|
await asyncio.to_thread(manager.close)
|
|
|
return
|
|
|
if scope["method"] == "GET" and path == "/internal/v1/health":
|
|
|
await asyncio.to_thread(self.store.get, "engine", "readiness")
|
|
|
maintenance = await asyncio.to_thread(self.updater.active_job)
|
|
|
await json_response(send, 200, {
|
|
|
"status": "ok",
|
|
|
"accepting_mutations": maintenance is None,
|
|
|
})
|
|
|
return
|
|
|
cloud_template_match = re.fullmatch(
|
|
|
r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|"
|
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
|
|
|
if cloud_template_match is not None:
|
|
|
resource = cloud_template_match[1]
|
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
|
await self._cloud_template_dispatch(
|
|
|
scope["method"], resource, body, send,
|
|
|
scope=scope, receive=receive)
|
|
|
return
|
|
|
user_iso_match = re.fullmatch(
|
|
|
r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|"
|
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?)", path)
|
|
|
if user_iso_match is not None:
|
|
|
resource = user_iso_match[1]
|
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
|
await self._user_iso_dispatch(
|
|
|
scope["method"], resource, body, delegated, send)
|
|
|
return
|
|
|
op_match = re.fullmatch(r"/internal/v1/operations/([A-Za-z0-9_-]{8,80})", path)
|
|
|
if scope["method"] == "GET" and op_match:
|
|
|
query = scope.get("query_string", b"")
|
|
|
if query not in {b"", b"reconcile=false"}:
|
|
|
raise EngineAuthError("操作查询参数无效", 400)
|
|
|
operation = await self._operation(
|
|
|
op_match[1], delegated, reconcile=query != b"reconcile=false")
|
|
|
await json_response(send, 200, self._public_operation(operation))
|
|
|
return
|
|
|
match = re.fullmatch(r"/internal/v1/hosts/([A-Za-z0-9][A-Za-z0-9_-]{0,63})/(.+)", path)
|
|
|
if match is None:
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
host_id, resource = match.groups()
|
|
|
if re.fullmatch(
|
|
|
r"user-isos(?:/import|/[0-9a-f]{32}|"
|
|
|
r"/import-jobs/[A-Za-z0-9_-]+)?", resource):
|
|
|
# User ISO ownership, quota, source bytes and deletion tombstones
|
|
|
# are engine-global. Keeping the historical host-prefixed alias
|
|
|
# reachable would let DELETE bypass replica GC and make the same
|
|
|
# catalog appear to have host-local semantics.
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
self._authorize_route(scope["method"], resource, delegated)
|
|
|
app = await self.registry.get(host_id)
|
|
|
# Host reads are observations only: they never acquire the engine-wide
|
|
|
# mutation/update lock and never repair state as a side effect. Cache
|
|
|
# cleanup belongs to the explicit delete workflow that created its
|
|
|
# durable tombstone, so unrelated host writes do not run it either.
|
|
|
if resource.endswith("/console-ticket"):
|
|
|
vm_id = resource.split("/")[1]
|
|
|
await self._ticket(host_id, vm_id, delegated, send)
|
|
|
return
|
|
|
operation = None
|
|
|
create = scope["method"] == "POST" and resource == "vms"
|
|
|
mutation = scope["method"] == "DELETE" and resource.startswith("vms/") or (
|
|
|
scope["method"] == "PUT" and bool(re.fullmatch(
|
|
|
rf"vms/{VM_ID}/firewall", resource))) or (
|
|
|
scope["method"] == "POST" and bool(re.fullmatch(
|
|
|
rf"vms/{VM_ID}/(?:actions|reinstall|network|settings|"
|
|
|
rf"(?:service|network)-(?:suspend|resume)|"
|
|
|
rf"rescue(?:/exit|/force-start)?|"
|
|
|
rf"snapshots(?:/[0-9a-f]{{32}}/restore)?)",
|
|
|
resource)))
|
|
|
if create or mutation:
|
|
|
if not OP_ID.fullmatch(delegated.get("operation_id", "")):
|
|
|
raise EngineAuthError("虚拟机操作缺少持久幂等编号", 403)
|
|
|
if create and not delegated.get("can_provision"):
|
|
|
raise EngineAuthError("虚拟机只能通过已确认套餐订单开通", 403)
|
|
|
operation, fresh = await self._reserve_operation(host_id, delegated, body,
|
|
|
resource=resource, method=scope["method"], target=raw_target(scope))
|
|
|
if operation.get("prevalidation_failed"):
|
|
|
await json_response(send, 422, self._public_operation(operation))
|
|
|
return
|
|
|
if not fresh:
|
|
|
if (operation["status"] in {"running", "pending", "uncertain"}
|
|
|
and (self._service_target_method(
|
|
|
operation.get("resource", ""))
|
|
|
or self._is_network_change_resource(
|
|
|
operation.get("resource", ""))
|
|
|
or self._is_firewall_resource(
|
|
|
operation.get("resource", ""))
|
|
|
or self._is_power_action_resource(
|
|
|
operation.get("resource", "")))):
|
|
|
operation = await self._mutation_operation(operation)
|
|
|
if operation["status"] == "succeeded":
|
|
|
await json_response(send, 201 if create else 200,
|
|
|
{**operation["result"], **self._public_operation(operation)})
|
|
|
else:
|
|
|
await json_response(send, 202, self._public_operation(operation))
|
|
|
return
|
|
|
if create:
|
|
|
delegated["vm_uuid"] = operation["vm_uuid"]
|
|
|
self.active_operations.add(operation["operation_id"])
|
|
|
child_scope = {**scope, "path": "/api/v1/" + resource,
|
|
|
"raw_path": ("/api/v1/" + resource).encode(), "root_path": ""}
|
|
|
if operation is not None and self._is_network_change_resource(resource):
|
|
|
def mark_network_mutation_started():
|
|
|
updated = {**operation, "network_mutation_started": True}
|
|
|
self.store.set(
|
|
|
"operation", operation["operation_id"],
|
|
|
json.dumps(updated))
|
|
|
operation.update(network_mutation_started=True)
|
|
|
|
|
|
child_scope["virt_cloud_mark_network_mutation_started"] = \
|
|
|
mark_network_mutation_started
|
|
|
if operation is None:
|
|
|
# Register before saving so all persisted hosts are restored for
|
|
|
# unattended sampling after restart, even if the HTTP caller dies.
|
|
|
if resource == "host" and scope["method"] == "PUT":
|
|
|
await asyncio.to_thread(self.store.set, "engine_host", host_id, json.dumps({"host_id": host_id}))
|
|
|
await app(child_scope, receive, send)
|
|
|
return
|
|
|
messages = []
|
|
|
async def capture(message):
|
|
|
messages.append(message)
|
|
|
try:
|
|
|
target_method = self._service_target_method(operation.get("resource", ""))
|
|
|
# Every VM mutation shares the same lock, not just billing target
|
|
|
# state. A start/reset/delete that timed out at PHP may still be
|
|
|
# executing here when expiry enforcement arrives. Serializing the
|
|
|
# later policy call guarantees that its desired state wins. The
|
|
|
# reinstall/rescue endpoints register their per-VM pending guard
|
|
|
# before returning 202; child endpoints reject every other
|
|
|
# mutation until those background jobs remove that guard.
|
|
|
async with self._target_lock(host_id, operation["vm_uuid"]):
|
|
|
if target_method == "service_suspend":
|
|
|
await self._capture_service_was_running_hint(
|
|
|
host_id, operation, delegated)
|
|
|
elif self._is_network_change_resource(
|
|
|
operation.get("resource", "")):
|
|
|
await self._capture_network_transition(
|
|
|
host_id, operation)
|
|
|
elif self._is_vm_delete_operation(operation):
|
|
|
await self._capture_delete_cleanup(
|
|
|
host_id, operation, delegated)
|
|
|
await app(child_scope, receive, capture)
|
|
|
status = next(item["status"] for item in messages if item["type"] == "http.response.start")
|
|
|
payload = json.loads(b"".join(item.get("body", b"") for item in messages
|
|
|
if item["type"] == "http.response.body"))
|
|
|
network_preflight_failed = (
|
|
|
self._is_network_change_resource(
|
|
|
operation.get("resource", ""))
|
|
|
and operation.get("network_mutation_started") is False
|
|
|
)
|
|
|
operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else (
|
|
|
"failed" if not create and (
|
|
|
status < 500 or network_preflight_failed) else "uncertain")
|
|
|
if operation["status"] == "succeeded":
|
|
|
operation["result"] = payload
|
|
|
elif operation["status"] == "pending":
|
|
|
operation["result"] = payload
|
|
|
operation["job_id"] = payload["job_id"]
|
|
|
else:
|
|
|
operation["error"] = payload.get("detail") or payload.get("error") or "虚拟机开通失败"
|
|
|
error_code = payload.get("error_code")
|
|
|
if (("/snapshots" in operation.get("resource", "")
|
|
|
and error_code in SNAPSHOT_PUBLIC_ERROR_CODES)
|
|
|
or (self._is_firewall_resource(
|
|
|
operation.get("resource", ""))
|
|
|
and error_code in FIREWALL_PUBLIC_ERROR_CODES)
|
|
|
or (operation.get("resource", "").endswith("/reinstall")
|
|
|
and error_code in REINSTALL_PUBLIC_ERROR_CODES)
|
|
|
or (network_preflight_failed
|
|
|
and error_code in FIREWALL_PUBLIC_ERROR_CODES)):
|
|
|
operation["error_code"] = error_code
|
|
|
if self._is_vm_delete_operation(operation):
|
|
|
cleanup_code = payload.get("error_code")
|
|
|
if cleanup_code in DELETE_PUBLIC_ERROR_CODES:
|
|
|
# These codes are emitted only after the child endpoint
|
|
|
# has crossed the irreversible domain-delete boundary.
|
|
|
# Persist that commit boundary so later polling can
|
|
|
# finish exact cleanup without replaying VM DELETE.
|
|
|
operation["cleanup_pending"] = "vm_delete"
|
|
|
operation["vm_delete_committed"] = True
|
|
|
if create:
|
|
|
await self._check_absent(operation)
|
|
|
operation["finished_at"] = int(time.time())
|
|
|
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
|
|
|
if operation["status"] in {"succeeded", "pending"}:
|
|
|
await json_response(send, status, {**payload, **self._public_operation(operation)})
|
|
|
else:
|
|
|
await json_response(send, status, {**payload, "operation_id": operation["operation_id"],
|
|
|
"status": operation["status"], "safe_to_refund": operation["safe_to_refund"]})
|
|
|
except BaseException as exc:
|
|
|
network_preflight_failed = (
|
|
|
self._is_network_change_resource(
|
|
|
operation.get("resource", ""))
|
|
|
and operation.get("network_mutation_started") is False
|
|
|
)
|
|
|
preflight_status = getattr(exc, "status", None)
|
|
|
if (isinstance(preflight_status, int)
|
|
|
and preflight_status < 500):
|
|
|
operation.update(
|
|
|
status="failed",
|
|
|
error=str(exc),
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
elif network_preflight_failed:
|
|
|
operation.update(
|
|
|
status="failed",
|
|
|
error="网络变更尚未开始;前置核对被中断",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
else:
|
|
|
operation["status"] = "uncertain"
|
|
|
operation["error"] = (
|
|
|
"防火墙结果不确定,请查询原操作核对,不要盲目重放"
|
|
|
if self._is_firewall_resource(operation.get("resource", ""))
|
|
|
else "开通结果不确定,请查询操作状态,不要重复创建"
|
|
|
)
|
|
|
with suppress(Exception):
|
|
|
await asyncio.to_thread(self.store.set, "operation", operation["operation_id"], json.dumps(operation))
|
|
|
raise
|
|
|
finally:
|
|
|
self.active_operations.discard(operation["operation_id"])
|
|
|
|
|
|
@staticmethod
|
|
|
def _cloud_upload_metadata(scope) -> tuple[str, str, str]:
|
|
|
query = scope.get("query_string", b"")
|
|
|
if (not isinstance(query, bytes) or len(query) > 2048
|
|
|
or re.search(rb"%(?![0-9A-Fa-f]{2})", query) is not None):
|
|
|
raise TemplateError("云镜像上传参数无效", 422)
|
|
|
try:
|
|
|
pairs = parse_qsl(
|
|
|
query.decode("ascii"),
|
|
|
keep_blank_values=True,
|
|
|
strict_parsing=True,
|
|
|
max_num_fields=3,
|
|
|
errors="strict",
|
|
|
)
|
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
|
raise TemplateError("云镜像上传参数无效", 422) from exc
|
|
|
if (len(pairs) != 3
|
|
|
or {key for key, _value in pairs}
|
|
|
!= {"name", "os_family", "os_version"}):
|
|
|
raise TemplateError("云镜像上传参数无效", 422)
|
|
|
values = dict(pairs)
|
|
|
name = validate_template_name(values["name"])
|
|
|
os_family, os_version = validate_template_metadata(
|
|
|
values["os_family"], values["os_version"])
|
|
|
return name, os_family, os_version
|
|
|
|
|
|
@staticmethod
|
|
|
def _single_upload_header(scope, name: bytes) -> str | None:
|
|
|
values = [value.decode("latin-1")
|
|
|
for key, value in scope.get("headers", [])
|
|
|
if key.lower() == name]
|
|
|
if len(values) > 1:
|
|
|
raise TemplateError("云镜像上传请求头重复", 400)
|
|
|
return values[0] if values else None
|
|
|
|
|
|
async def _upload_cloud_template(self, scope, receive, send) -> None:
|
|
|
runtime = self.registry.cloud_templates
|
|
|
name, os_family, os_version = self._cloud_upload_metadata(scope)
|
|
|
content_type = self._single_upload_header(scope, b"content-type")
|
|
|
if content_type is None or content_type.lower() != "application/octet-stream":
|
|
|
raise TemplateError(
|
|
|
"云镜像上传必须使用 application/octet-stream", 415)
|
|
|
content_encoding = self._single_upload_header(scope, b"content-encoding")
|
|
|
if content_encoding is not None and content_encoding.lower() != "identity":
|
|
|
raise TemplateError("云镜像上传内容不能压缩", 415)
|
|
|
|
|
|
declared_raw = self._single_upload_header(scope, b"content-length")
|
|
|
if declared_raw is not None and re.fullmatch(r"[0-9]+", declared_raw) is None:
|
|
|
raise TemplateError("云镜像上传长度无效", 400)
|
|
|
declared = int(declared_raw) if declared_raw is not None else None
|
|
|
if declared is not None and declared < MIN_DOWNLOAD_BYTES:
|
|
|
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
|
|
|
if declared is not None and declared > MAX_IMAGE_BYTES:
|
|
|
raise TemplateError("云镜像超过 4 GiB 限制", 413)
|
|
|
|
|
|
async with self._update_lock():
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
raise EngineAuthError(
|
|
|
"面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
|
reservation = await asyncio.to_thread(
|
|
|
runtime.reserve_upload, name, os_family, os_version)
|
|
|
reservation_id = reservation["reservation_id"]
|
|
|
maximum = reservation["max_bytes"]
|
|
|
if declared is not None and declared > maximum:
|
|
|
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
|
|
|
raise TemplateError("面板云镜像持久目录空间不足", 507)
|
|
|
|
|
|
image_path: Path | None = None
|
|
|
handed_off = False
|
|
|
writer: _CloudUploadWriter | None = None
|
|
|
try:
|
|
|
writer = await asyncio.to_thread(
|
|
|
_CloudUploadWriter, reservation["directory"])
|
|
|
image_path = writer.path
|
|
|
checksum = hashlib.sha256()
|
|
|
size = 0
|
|
|
deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS
|
|
|
try:
|
|
|
while True:
|
|
|
remaining = deadline - time.monotonic()
|
|
|
if remaining <= 0:
|
|
|
raise TemplateError("云镜像上传超时", 408)
|
|
|
try:
|
|
|
message = await asyncio.wait_for(
|
|
|
receive(),
|
|
|
timeout=min(CLOUD_UPLOAD_IDLE_SECONDS, remaining),
|
|
|
)
|
|
|
except TimeoutError as exc:
|
|
|
raise TemplateError("云镜像上传超时", 408) from exc
|
|
|
if message["type"] == "http.disconnect":
|
|
|
raise TemplateError("云镜像上传已中断", 400)
|
|
|
if message["type"] != "http.request":
|
|
|
raise TemplateError("云镜像上传请求无效", 400)
|
|
|
chunk = message.get("body", b"")
|
|
|
size += len(chunk)
|
|
|
if declared is not None and size > declared:
|
|
|
raise TemplateError("云镜像上传内容超过声明长度", 400)
|
|
|
if size > maximum:
|
|
|
if maximum < MAX_IMAGE_BYTES:
|
|
|
raise TemplateError(
|
|
|
"面板云镜像持久目录空间不足", 507)
|
|
|
raise TemplateError("云镜像超过 4 GiB 限制", 413)
|
|
|
view = memoryview(chunk)
|
|
|
for offset in range(0, len(view), CLOUD_UPLOAD_WRITE_BYTES):
|
|
|
if time.monotonic() >= deadline:
|
|
|
raise TemplateError("云镜像上传超时", 408)
|
|
|
part = view[offset:offset + CLOUD_UPLOAD_WRITE_BYTES]
|
|
|
await asyncio.to_thread(writer.write, part)
|
|
|
checksum.update(part)
|
|
|
if not message.get("more_body", False):
|
|
|
break
|
|
|
await asyncio.to_thread(writer.sync_and_close)
|
|
|
except OSError as exc:
|
|
|
if exc.errno in {errno.ENOSPC, errno.EDQUOT}:
|
|
|
raise TemplateError(
|
|
|
"面板云镜像持久目录空间不足", 507) from exc
|
|
|
raise
|
|
|
if declared is not None and size != declared:
|
|
|
raise TemplateError("云镜像上传内容不完整", 400)
|
|
|
if size < MIN_DOWNLOAD_BYTES:
|
|
|
raise TemplateError("云镜像文件不能小于 1 MiB", 422)
|
|
|
validation = scope.get("engine_upload_validation", {})
|
|
|
expected_digest = scope.get("engine_upload_digest", "")
|
|
|
digest = checksum.hexdigest()
|
|
|
if (not validation.get("complete")
|
|
|
or not validation.get("valid")
|
|
|
or not hmac.compare_digest(digest, expected_digest)):
|
|
|
raise EngineAuthError("云镜像摘要与签名不匹配", 401)
|
|
|
virtual_size = await asyncio.to_thread(
|
|
|
verify_qcow2_image, image_path)
|
|
|
# Recheck the update state at the commit boundary. An upload
|
|
|
# admitted before a service transition must not enqueue work
|
|
|
# against code that is being replaced.
|
|
|
async with self._update_lock():
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
raise EngineAuthError(
|
|
|
"面板正在更新,暂时不能执行虚拟化写操作", 409)
|
|
|
result = await asyncio.to_thread(
|
|
|
runtime.queue_upload,
|
|
|
reservation_id,
|
|
|
image_path,
|
|
|
digest,
|
|
|
virtual_size,
|
|
|
)
|
|
|
handed_off = True
|
|
|
await json_response(send, 202, result)
|
|
|
finally:
|
|
|
if not handed_off:
|
|
|
if writer is not None:
|
|
|
with suppress(OSError):
|
|
|
await asyncio.to_thread(writer.close)
|
|
|
if image_path is not None:
|
|
|
with suppress(OSError):
|
|
|
await asyncio.to_thread(image_path.unlink, missing_ok=True)
|
|
|
await asyncio.to_thread(runtime.cancel_upload, reservation_id)
|
|
|
|
|
|
async def _cloud_template_dispatch(
|
|
|
self, method: str, resource: str, body: bytes | None, send,
|
|
|
*, scope=None, receive=None) -> None:
|
|
|
runtime = self.registry.cloud_templates
|
|
|
try:
|
|
|
if method == "GET" and resource == "vm-templates":
|
|
|
records = await asyncio.to_thread(runtime.list)
|
|
|
await json_response(send, 200, {"templates": records})
|
|
|
return
|
|
|
if method == "POST" and resource == "vm-templates/import":
|
|
|
from .app import ImportTemplateBody
|
|
|
from pydantic import ValidationError
|
|
|
try:
|
|
|
value = ImportTemplateBody.model_validate_json(body or b"")
|
|
|
except ValidationError as exc:
|
|
|
raise TemplateError("云镜像导入参数无效", 422) from exc
|
|
|
result = await asyncio.to_thread(
|
|
|
runtime.queue_import, value.url, value.name, value.sha256)
|
|
|
await json_response(send, 202, result)
|
|
|
return
|
|
|
if method == "POST" and resource == "vm-templates/upload":
|
|
|
if scope is None or receive is None:
|
|
|
raise EngineAuthError("云镜像上传请求无效", 400)
|
|
|
await self._upload_cloud_template(scope, receive, send)
|
|
|
return
|
|
|
job_match = re.fullmatch(
|
|
|
r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource)
|
|
|
if method == "GET" and job_match is not None:
|
|
|
result = await asyncio.to_thread(
|
|
|
runtime.get_job, job_match[1])
|
|
|
await json_response(send, 200, result)
|
|
|
return
|
|
|
template_match = re.fullmatch(
|
|
|
r"vm-templates/([A-Za-z0-9_-]+)", resource)
|
|
|
if method == "PATCH" and template_match is not None:
|
|
|
from .app import RenameTemplateBody
|
|
|
from pydantic import ValidationError
|
|
|
try:
|
|
|
value = RenameTemplateBody.model_validate_json(body or b"")
|
|
|
except ValidationError as exc:
|
|
|
raise TemplateError("云镜像名称无效", 422) from exc
|
|
|
record = await asyncio.to_thread(
|
|
|
runtime.rename, template_match[1], value.name)
|
|
|
await json_response(send, 200, {"template": record})
|
|
|
return
|
|
|
if method == "DELETE" and template_match is not None:
|
|
|
from .app import DeleteTemplateBody
|
|
|
from pydantic import ValidationError
|
|
|
try:
|
|
|
value = DeleteTemplateBody.model_validate_json(body or b"")
|
|
|
except ValidationError as exc:
|
|
|
raise TemplateError("云镜像删除参数无效", 422) from exc
|
|
|
template_id = template_match[1]
|
|
|
tombstone_key, tombstone = await asyncio.to_thread(
|
|
|
self._delete_cloud_template_source,
|
|
|
template_id, value.created_at)
|
|
|
if not self._wake_cache_cleanup():
|
|
|
for host_id in tombstone["pending_hosts"]:
|
|
|
self._background(
|
|
|
self._cleanup_cloud_template_tombstone_on_host(
|
|
|
tombstone_key, tombstone, host_id))
|
|
|
await json_response(send, 200, {
|
|
|
"deleted": True,
|
|
|
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
|
|
|
})
|
|
|
return
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
except TemplateError as exc:
|
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
|
|
|
async def _user_iso_dispatch(self, method: str, resource: str,
|
|
|
body: bytes | None, delegated: dict, send) -> None:
|
|
|
runtime = self.registry.user_isos
|
|
|
user_id = delegated["user_id"]
|
|
|
try:
|
|
|
if method == "GET" and resource == "user-isos":
|
|
|
result = await asyncio.to_thread(runtime.list_for, user_id)
|
|
|
await json_response(send, 200, result)
|
|
|
return
|
|
|
if method == "POST" and resource == "user-isos/import":
|
|
|
from .app import ImportUserISOBody
|
|
|
from pydantic import ValidationError
|
|
|
try:
|
|
|
value = ImportUserISOBody.model_validate_json(body or b"")
|
|
|
except ValidationError as exc:
|
|
|
raise TemplateError("ISO 导入参数无效", 422) from exc
|
|
|
result = await asyncio.to_thread(
|
|
|
runtime.queue_import, user_id, value.url, value.name,
|
|
|
value.sha256)
|
|
|
await json_response(send, 202, result)
|
|
|
return
|
|
|
job_match = re.fullmatch(
|
|
|
r"user-isos/import-jobs/([A-Za-z0-9_-]+)", resource)
|
|
|
if method == "GET" and job_match is not None:
|
|
|
result = await asyncio.to_thread(
|
|
|
runtime.get_job_for, user_id, job_match[1])
|
|
|
await json_response(send, 200, result)
|
|
|
return
|
|
|
iso_match = re.fullmatch(r"user-isos/([0-9a-f]{32})", resource)
|
|
|
if iso_match is not None and method == "GET":
|
|
|
record = await asyncio.to_thread(
|
|
|
runtime.get_for, user_id, iso_match[1])
|
|
|
await json_response(
|
|
|
send, 200, {"iso": runtime.public_record(record)})
|
|
|
return
|
|
|
if iso_match is not None and method == "DELETE":
|
|
|
iso_id = iso_match[1]
|
|
|
tombstone_key, tombstone = await asyncio.to_thread(
|
|
|
self._delete_user_iso_source, user_id, iso_id)
|
|
|
if not self._wake_cache_cleanup():
|
|
|
for host_id in tombstone["pending_hosts"]:
|
|
|
self._background(
|
|
|
self._cleanup_user_iso_tombstone_on_host(
|
|
|
tombstone_key, tombstone, host_id))
|
|
|
await json_response(send, 200, {
|
|
|
"deleted": True,
|
|
|
"cache_cleanup_pending": bool(tombstone["pending_hosts"]),
|
|
|
})
|
|
|
return
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
except TemplateError as exc:
|
|
|
await json_response(send, exc.status, {"error": str(exc)})
|
|
|
|
|
|
def _delete_cloud_template_source(
|
|
|
self, template_id: str,
|
|
|
expected_created_at: int) -> tuple[str, dict]:
|
|
|
"""Commit global source deletion and durable per-host cache GC."""
|
|
|
runtime = self.registry.cloud_templates
|
|
|
with runtime.lock:
|
|
|
raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id)
|
|
|
if raw is not None:
|
|
|
tombstone = self._decode_cloud_template_tombstone(
|
|
|
template_id, raw)
|
|
|
if tombstone["template_created_at"] != expected_created_at:
|
|
|
raise TemplateError(
|
|
|
"云镜像版本已变化,请刷新后重试", 409)
|
|
|
if tombstone["source_deleted"]:
|
|
|
return template_id, tombstone
|
|
|
|
|
|
try:
|
|
|
record = runtime.get(template_id)
|
|
|
except TemplateError as exc:
|
|
|
if raw is None or exc.status != 404:
|
|
|
raise
|
|
|
tombstone["source_deleted"] = True
|
|
|
self.store.set(
|
|
|
CACHE_TOMBSTONE_KIND, template_id,
|
|
|
json.dumps(tombstone, separators=(",", ":")),
|
|
|
)
|
|
|
return template_id, tombstone
|
|
|
|
|
|
source_volume_name = record.get("volume_name")
|
|
|
if source_volume_name != template_volume_name(template_id):
|
|
|
raise TemplateError("云镜像目录记录损坏,无法安全删除", 500)
|
|
|
if record.get("created_at") != expected_created_at:
|
|
|
raise TemplateError("云镜像版本已变化,请刷新后重试", 409)
|
|
|
host_ids = sorted(
|
|
|
set(self.store.list("engine_host"))
|
|
|
| set(self.registry.components)
|
|
|
)
|
|
|
tombstone = {
|
|
|
"deletion_id": uuid.uuid4().hex,
|
|
|
"template_id": template_id,
|
|
|
"template_created_at": expected_created_at,
|
|
|
"source_volume_name": source_volume_name,
|
|
|
"pending_hosts": host_ids,
|
|
|
"source_deleted": False,
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
encoded = json.dumps(tombstone, separators=(",", ":"))
|
|
|
self.store.set(CACHE_TOMBSTONE_KIND, template_id, encoded)
|
|
|
try:
|
|
|
# Re-enters runtime.lock, which is intentionally an RLock.
|
|
|
runtime.delete(template_id)
|
|
|
except Exception:
|
|
|
# A database DELETE acknowledgement can be lost before or
|
|
|
# after commit. The catalog alone is not sufficient proof of
|
|
|
# rollback because source unlink happens first. Reconcile both
|
|
|
# durable halves: only an intact catalog+source pair cancels
|
|
|
# this marker; a missing source completes retirement and keeps
|
|
|
# host GC durable.
|
|
|
if self.store.get(
|
|
|
CACHE_TOMBSTONE_KIND, template_id) == encoded:
|
|
|
reconciled = runtime._reconcile_cache_tombstone(
|
|
|
template_id, tombstone.copy())
|
|
|
if (reconciled is not None
|
|
|
and reconciled["source_deleted"]):
|
|
|
return template_id, reconciled
|
|
|
raise
|
|
|
tombstone["source_deleted"] = True
|
|
|
self.store.set(
|
|
|
CACHE_TOMBSTONE_KIND, template_id,
|
|
|
json.dumps(tombstone, separators=(",", ":")),
|
|
|
)
|
|
|
return template_id, tombstone
|
|
|
|
|
|
@staticmethod
|
|
|
def _decode_cloud_template_tombstone(key: str, raw: str) -> dict:
|
|
|
try:
|
|
|
value = json.loads(raw)
|
|
|
deletion_id = value["deletion_id"]
|
|
|
template_id = value["template_id"]
|
|
|
template_created_at = value["template_created_at"]
|
|
|
pending = value["pending_hosts"]
|
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
|
raise ValueError("cloud template tombstone is corrupt") from exc
|
|
|
try:
|
|
|
expected_volume_name = template_volume_name(template_id)
|
|
|
except TemplateError as exc:
|
|
|
raise ValueError("cloud template tombstone is corrupt") from exc
|
|
|
if (not isinstance(deletion_id, str)
|
|
|
or not re.fullmatch(r"[0-9a-f]{32}", deletion_id)
|
|
|
or key != template_id
|
|
|
or isinstance(template_created_at, bool)
|
|
|
or not isinstance(template_created_at, int)
|
|
|
or template_created_at < 1
|
|
|
or value.get("source_volume_name") != expected_volume_name
|
|
|
or not isinstance(pending, list)
|
|
|
or any(not isinstance(host_id, str)
|
|
|
or not HOST_ID.fullmatch(host_id)
|
|
|
for host_id in pending)
|
|
|
or len(set(pending)) != len(pending)
|
|
|
or not isinstance(value.get("source_deleted"), bool)):
|
|
|
raise ValueError("cloud template tombstone is corrupt")
|
|
|
return value
|
|
|
|
|
|
async def _cleanup_cloud_template_tombstone_on_host(
|
|
|
self, key: str, tombstone: dict, host_id: str) -> None:
|
|
|
if (not tombstone.get("source_deleted")
|
|
|
or host_id not in tombstone["pending_hosts"]):
|
|
|
return
|
|
|
try:
|
|
|
await self.registry.get(host_id)
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return
|
|
|
service = self._service(profile)
|
|
|
await asyncio.to_thread(
|
|
|
self._cleanup_cloud_template_tombstone_on_host_sync,
|
|
|
key, tombstone["deletion_id"], host_id, service, components)
|
|
|
except Exception as exc:
|
|
|
LOG.info(
|
|
|
"Cloud template cache cleanup deferred: host=%s type=%s",
|
|
|
host_id, type(exc).__name__)
|
|
|
|
|
|
def _cleanup_cloud_template_tombstone_on_host_sync(
|
|
|
self, key: str, deletion_id: str, host_id: str,
|
|
|
service, components: EngineComponents) -> None:
|
|
|
"""Revalidate the deletion generation at the libvirt commit point.
|
|
|
|
|
|
Duplicate cleanup tasks can outlive the tombstone snapshot that
|
|
|
scheduled them. Holding the shared runtime lock across the final
|
|
|
re-read, libvirt deletion and marker update prevents an obsolete task
|
|
|
from deleting a replica created by a same-content reimport.
|
|
|
"""
|
|
|
runtime = self.registry.cloud_templates
|
|
|
with runtime.lock:
|
|
|
raw = self.store.get(CACHE_TOMBSTONE_KIND, key)
|
|
|
if raw is None:
|
|
|
return
|
|
|
value = self._decode_cloud_template_tombstone(key, raw)
|
|
|
if (value["deletion_id"] != deletion_id
|
|
|
or not value["source_deleted"]
|
|
|
or host_id not in value["pending_hosts"]):
|
|
|
return
|
|
|
try:
|
|
|
service.delete_cloud_template_replicas(
|
|
|
value["source_volume_name"])
|
|
|
except Exception as exc:
|
|
|
from .libvirt_service import VMError
|
|
|
# Only an accessible pool with no matching volumes is an
|
|
|
# idempotent success. A missing/misconfigured storage pool is
|
|
|
# also reported as 404 by libvirt_service and must keep this
|
|
|
# host pending so an old replica cannot escape reclamation.
|
|
|
if (not isinstance(exc, VMError)
|
|
|
or exc.status != 404
|
|
|
or exc.error_code
|
|
|
!= "cloud_template_cache_not_found"):
|
|
|
raise
|
|
|
components.store.delete(
|
|
|
"cloud_template_replica", value["template_id"])
|
|
|
value["pending_hosts"] = [
|
|
|
candidate for candidate in value["pending_hosts"]
|
|
|
if candidate != host_id
|
|
|
]
|
|
|
# Retain the terminal record so a response lost after source
|
|
|
# deletion remains retryable until an intentional reimport.
|
|
|
self.store.set(
|
|
|
CACHE_TOMBSTONE_KIND, key,
|
|
|
json.dumps(value, separators=(",", ":")),
|
|
|
)
|
|
|
|
|
|
async def _cleanup_host_cloud_template_tombstones(
|
|
|
self, host_id: str) -> None:
|
|
|
rows = await asyncio.to_thread(
|
|
|
self.store.list, CACHE_TOMBSTONE_KIND)
|
|
|
for key, raw in rows.items():
|
|
|
try:
|
|
|
tombstone = self._decode_cloud_template_tombstone(key, raw)
|
|
|
except ValueError:
|
|
|
LOG.error(
|
|
|
"Corrupt cloud template cache cleanup record: key=%s", key)
|
|
|
continue
|
|
|
if host_id in tombstone["pending_hosts"]:
|
|
|
await self._cleanup_cloud_template_tombstone_on_host(
|
|
|
key, tombstone, host_id)
|
|
|
|
|
|
def _delete_user_iso_source(
|
|
|
self, user_id: int, iso_id: str) -> tuple[str, dict]:
|
|
|
"""Commit one source deletion and its remote-GC tombstone atomically.
|
|
|
|
|
|
The shared runtime lock serializes the whole transition. In
|
|
|
particular, a second DELETE can only observe the first request's
|
|
|
committed tombstone; it must never remove that durable cleanup record
|
|
|
after the catalog has disappeared.
|
|
|
"""
|
|
|
runtime = self.registry.user_isos
|
|
|
key = f"{user_id}:{iso_id}"
|
|
|
with runtime.lock:
|
|
|
raw = self.store.get("user_iso_tombstone", key)
|
|
|
if raw is not None:
|
|
|
tombstone = self._decode_user_iso_tombstone(key, raw)
|
|
|
if tombstone["source_deleted"]:
|
|
|
return key, tombstone
|
|
|
|
|
|
try:
|
|
|
record = runtime.get_for(user_id, iso_id)
|
|
|
except TemplateError as exc:
|
|
|
if raw is None or exc.status != 404:
|
|
|
raise
|
|
|
# The source/catalog removal committed but persisting the
|
|
|
# final flag failed. Preserve and advance the existing GC
|
|
|
# record instead of erasing it on a repeated DELETE.
|
|
|
tombstone["source_deleted"] = True
|
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
|
tombstone, separators=(",", ":")))
|
|
|
if not tombstone["pending_hosts"]:
|
|
|
self.store.delete("user_iso_tombstone", key)
|
|
|
return key, tombstone
|
|
|
host_ids = self._user_iso_replica_hosts(user_id, record)
|
|
|
tombstone = {
|
|
|
"user_id": user_id,
|
|
|
"iso_id": iso_id,
|
|
|
"volume_name": record["volume_name"],
|
|
|
"pending_hosts": host_ids,
|
|
|
"source_deleted": False,
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
encoded = json.dumps(tombstone, separators=(",", ":"))
|
|
|
self.store.set("user_iso_tombstone", key, encoded)
|
|
|
try:
|
|
|
# Re-enters runtime.lock; it is intentionally an RLock.
|
|
|
runtime.delete_for(user_id, iso_id)
|
|
|
except Exception:
|
|
|
# Only erase the exact provisional record written above. A
|
|
|
# later committed tombstone must never be removed by a failed
|
|
|
# or repeated request.
|
|
|
if self.store.get("user_iso_tombstone", key) == encoded:
|
|
|
self.store.delete("user_iso_tombstone", key)
|
|
|
raise
|
|
|
tombstone["source_deleted"] = True
|
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
|
tombstone, separators=(",", ":")))
|
|
|
if not host_ids:
|
|
|
self.store.delete("user_iso_tombstone", key)
|
|
|
return key, tombstone
|
|
|
|
|
|
def _user_iso_replica_hosts(self, user_id: int,
|
|
|
record: dict) -> list[str]:
|
|
|
replica_key = f"{user_id}:{record['id']}"
|
|
|
result = []
|
|
|
host_ids = set(self.store.list("engine_host")) | set(self.registry.components)
|
|
|
for host_id in host_ids:
|
|
|
raw = HostStore(self.store, host_id).get(
|
|
|
"user_iso_replica", replica_key)
|
|
|
if raw is None:
|
|
|
continue
|
|
|
# The exact owner/ISO marker key is already proof that this host
|
|
|
# may contain the deterministic volume. Reuse requires a strict
|
|
|
# ready marker, but deletion must fail safe for old or partially
|
|
|
# damaged marker schemas as well.
|
|
|
result.append(host_id)
|
|
|
return sorted(result)
|
|
|
|
|
|
@staticmethod
|
|
|
def _decode_user_iso_tombstone(key: str, raw: str) -> dict:
|
|
|
from .user_isos import user_iso_volume_name
|
|
|
try:
|
|
|
value = json.loads(raw)
|
|
|
user_id = value["user_id"]
|
|
|
iso_id = value["iso_id"]
|
|
|
pending = value["pending_hosts"]
|
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
|
raise ValueError("user ISO tombstone is corrupt") from exc
|
|
|
if (isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1
|
|
|
or key != f"{user_id}:{iso_id}"
|
|
|
or value.get("volume_name") != user_iso_volume_name(iso_id)
|
|
|
or not isinstance(pending, list)
|
|
|
or any(not isinstance(host_id, str) or not HOST_ID.fullmatch(host_id)
|
|
|
for host_id in pending)
|
|
|
or len(set(pending)) != len(pending)
|
|
|
or not isinstance(value.get("source_deleted"), bool)):
|
|
|
raise ValueError("user ISO tombstone is corrupt")
|
|
|
return value
|
|
|
|
|
|
async def _cleanup_user_iso_tombstone_on_host(
|
|
|
self, key: str, tombstone: dict, host_id: str) -> None:
|
|
|
if not tombstone.get("source_deleted") or host_id not in tombstone["pending_hosts"]:
|
|
|
return
|
|
|
try:
|
|
|
await self.registry.get(host_id)
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return
|
|
|
service = self._service(profile)
|
|
|
try:
|
|
|
await asyncio.to_thread(
|
|
|
service.delete_user_iso, tombstone["user_id"],
|
|
|
tombstone["volume_name"])
|
|
|
except Exception as exc:
|
|
|
from .libvirt_service import VMError
|
|
|
# A root pool/configuration 404 is not proof that the exact
|
|
|
# per-user volume is absent. Keep the tombstone until the
|
|
|
# service returns the narrow idempotency code.
|
|
|
if (not isinstance(exc, VMError)
|
|
|
or exc.status != 404
|
|
|
or exc.error_code != "user_iso_volume_not_found"):
|
|
|
LOG.info(
|
|
|
"User ISO cache cleanup remains pending: host=%s type=%s",
|
|
|
host_id, type(exc).__name__)
|
|
|
return
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "user_iso_replica",
|
|
|
f"{tombstone['user_id']}:{tombstone['iso_id']}")
|
|
|
await asyncio.to_thread(
|
|
|
self._complete_user_iso_tombstone_host, key, host_id)
|
|
|
except Exception as exc:
|
|
|
LOG.info("User ISO cache cleanup deferred: host=%s type=%s",
|
|
|
host_id, type(exc).__name__)
|
|
|
|
|
|
def _complete_user_iso_tombstone_host(self, key: str, host_id: str) -> None:
|
|
|
runtime = self.registry.user_isos
|
|
|
with runtime.lock:
|
|
|
raw = self.store.get("user_iso_tombstone", key)
|
|
|
if raw is None:
|
|
|
return
|
|
|
value = self._decode_user_iso_tombstone(key, raw)
|
|
|
value["pending_hosts"] = [
|
|
|
candidate for candidate in value["pending_hosts"]
|
|
|
if candidate != host_id
|
|
|
]
|
|
|
if value["pending_hosts"]:
|
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
|
value, separators=(",", ":")))
|
|
|
else:
|
|
|
self.store.delete("user_iso_tombstone", key)
|
|
|
|
|
|
async def _cleanup_host_user_iso_tombstones(self, host_id: str) -> None:
|
|
|
rows = await asyncio.to_thread(self.store.list, "user_iso_tombstone")
|
|
|
for key, raw in rows.items():
|
|
|
try:
|
|
|
tombstone = self._decode_user_iso_tombstone(key, raw)
|
|
|
except ValueError:
|
|
|
LOG.error("Corrupt user ISO cache cleanup record: key=%s", key)
|
|
|
continue
|
|
|
if host_id in tombstone["pending_hosts"]:
|
|
|
await self._cleanup_user_iso_tombstone_on_host(
|
|
|
key, tombstone, host_id)
|
|
|
|
|
|
@staticmethod
|
|
|
def _update_object(body: bytes | None, *, empty: bool = False) -> dict:
|
|
|
if (body is None or body == b"") and empty:
|
|
|
return {}
|
|
|
try:
|
|
|
value = json.loads(body or b"")
|
|
|
except (TypeError, ValueError, UnicodeError) as exc:
|
|
|
raise UpdateError("在线更新参数必须是 JSON 对象", 422) from exc
|
|
|
if not isinstance(value, dict):
|
|
|
raise UpdateError("在线更新参数必须是 JSON 对象", 422)
|
|
|
return value
|
|
|
|
|
|
@staticmethod
|
|
|
def _public_update_job(job: dict) -> dict:
|
|
|
allowed = {
|
|
|
"job_id", "status", "phase", "created_at", "started_at", "finished_at",
|
|
|
"updated", "old_commit", "new_commit", "rollback_ref", "error",
|
|
|
}
|
|
|
return {key: value for key, value in job.items() if key in allowed}
|
|
|
|
|
|
def _long_running_host_job(self) -> bool:
|
|
|
cloud_templates = getattr(self.registry, "cloud_templates", None)
|
|
|
if (self.maintenance_tasks
|
|
|
or self.registry.user_isos.has_active_jobs()
|
|
|
or (cloud_templates is not None
|
|
|
and cloud_templates.has_active_jobs())):
|
|
|
return True
|
|
|
for components in self.registry.components.values():
|
|
|
kinds = [
|
|
|
"rescue_import_job", "reinstall_job", "rescue_job",
|
|
|
"snapshot_job",
|
|
|
]
|
|
|
# A child app without the shared runtime can still run the legacy
|
|
|
# host-scoped importer. Production children use the global runtime;
|
|
|
# stale pre-upgrade import_job rows must not block updates forever.
|
|
|
if components.cloud_templates is None:
|
|
|
kinds.append("import_job")
|
|
|
for kind in kinds:
|
|
|
for raw in components.store.list(kind).values():
|
|
|
try:
|
|
|
status = json.loads(raw).get("status")
|
|
|
except (AttributeError, TypeError, ValueError):
|
|
|
return True
|
|
|
if status in {"queued", "running", "pending"}:
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
async def _update_dispatch(self, method: str, path: str, body: bytes | None,
|
|
|
delegated: dict, send) -> None:
|
|
|
if not (delegated.get("is_admin") and delegated.get("is_super_admin")
|
|
|
and delegated.get("can_manage")):
|
|
|
raise EngineAuthError("仅超级管理员可以管理在线更新", 403)
|
|
|
|
|
|
if path == "/internal/v1/update/settings" and method == "GET":
|
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
|
|
|
return
|
|
|
if path == "/internal/v1/update/settings" and method == "PUT":
|
|
|
value = self._update_object(body)
|
|
|
if set(value) - {"access_token", "clear_token"}:
|
|
|
raise UpdateError("在线更新设置包含未知字段", 422)
|
|
|
clear = value.get("clear_token", False)
|
|
|
token = value.get("access_token")
|
|
|
if not isinstance(clear, bool) or (token is not None and not isinstance(token, str)):
|
|
|
raise UpdateError("在线更新设置格式无效", 422)
|
|
|
if clear and token not in {None, ""}:
|
|
|
raise UpdateError("不能同时保存和删除访问令牌", 422)
|
|
|
if clear:
|
|
|
await asyncio.to_thread(self.updater.clear_token)
|
|
|
elif token not in {None, ""}:
|
|
|
await asyncio.to_thread(self.updater.set_token, token)
|
|
|
elif "access_token" not in value:
|
|
|
raise UpdateError("没有需要保存的在线更新设置", 422)
|
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.settings))
|
|
|
return
|
|
|
if path == "/internal/v1/update/check" and method == "POST":
|
|
|
value = self._update_object(body, empty=True)
|
|
|
if value:
|
|
|
raise UpdateError("检查更新不接受额外参数", 422)
|
|
|
if await asyncio.to_thread(self.updater.active_job):
|
|
|
raise UpdateError("已有更新任务正在执行", 409)
|
|
|
await json_response(send, 200, await asyncio.to_thread(self.updater.check))
|
|
|
return
|
|
|
if path == "/internal/v1/update/apply" and method == "POST":
|
|
|
value = self._update_object(body)
|
|
|
if set(value) != {"expected_current_commit", "expected_remote_commit"}:
|
|
|
raise UpdateError("请先检查更新并提交已确认的版本", 422)
|
|
|
if not all(isinstance(value[key], str) for key in value):
|
|
|
raise UpdateError("更新版本标识无效", 422)
|
|
|
async with self._update_lock():
|
|
|
if self.active_operations or await asyncio.to_thread(self._long_running_host_job):
|
|
|
raise UpdateError("仍有虚拟机任务正在执行,请等待任务完成后再更新", 409)
|
|
|
job = await asyncio.to_thread(
|
|
|
self.updater.create_job,
|
|
|
expected_current_commit=value["expected_current_commit"],
|
|
|
expected_remote_commit=value["expected_remote_commit"],
|
|
|
)
|
|
|
try:
|
|
|
await asyncio.to_thread(self.update_trigger)
|
|
|
job = await self._wait_update_worker_started(job["job_id"])
|
|
|
except UpdateServiceUncertain:
|
|
|
raise
|
|
|
except UpdateError as exc:
|
|
|
await asyncio.to_thread(
|
|
|
self.updater.fail_job, job["job_id"],
|
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
|
|
|
)
|
|
|
raise exc
|
|
|
except Exception as exc:
|
|
|
await asyncio.to_thread(
|
|
|
self.updater.fail_job, job["job_id"],
|
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装",
|
|
|
)
|
|
|
raise UpdateError(
|
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
|
|
|
) from exc
|
|
|
await json_response(send, 202, self._public_update_job(job))
|
|
|
return
|
|
|
job_match = re.fullmatch(
|
|
|
r"/internal/v1/update/jobs/([A-Za-z0-9_-]{16,64})", path)
|
|
|
if method == "GET" and job_match:
|
|
|
job = await asyncio.to_thread(self.updater.get_job, job_match[1])
|
|
|
if job is None:
|
|
|
raise UpdateError("更新任务不存在", 404)
|
|
|
await json_response(send, 200, self._public_update_job(job))
|
|
|
return
|
|
|
raise EngineAuthError("内部资源不存在", 404)
|
|
|
|
|
|
async def _wait_update_worker_started(self, job_id: str) -> dict:
|
|
|
"""Confirm the one-shot claimed the durable job, not merely its queue."""
|
|
|
deadline = time.monotonic() + 15
|
|
|
while time.monotonic() < deadline:
|
|
|
job = await asyncio.to_thread(self.updater.get_job, job_id)
|
|
|
if job is None:
|
|
|
raise UpdateError("独立更新服务未生成任务状态", 503)
|
|
|
if job.get("status") in {"running", "succeeded"}:
|
|
|
return job
|
|
|
if job.get("status") == "failed":
|
|
|
raise UpdateError("独立更新服务启动后失败,请查看更新服务日志", 503)
|
|
|
if job.get("status") == "blocked":
|
|
|
raise UpdateServiceUncertain(
|
|
|
"更新任务中断且维护锁仍在生效,请通过 SSH 检查更新服务和回滚引用",
|
|
|
503,
|
|
|
)
|
|
|
await asyncio.sleep(0.1)
|
|
|
# Stop any very-late process before releasing the maintenance gate and
|
|
|
# marking the job failed. This prevents an unclaimed worker from
|
|
|
# changing code after ordinary VM mutations have resumed.
|
|
|
await asyncio.to_thread(self._stop_update_service)
|
|
|
raise UpdateError("独立更新服务未在限定时间内启动,请查看 systemd 日志", 503)
|
|
|
|
|
|
@staticmethod
|
|
|
def _start_update_service() -> None:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
["/usr/bin/systemctl", "--no-block", "start", "virt-cloud-update.service"],
|
|
|
cwd="/",
|
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=15,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
|
raise UpdateError(
|
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503
|
|
|
) from exc
|
|
|
if result.returncode != 0:
|
|
|
raise UpdateError(
|
|
|
"独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503)
|
|
|
|
|
|
@staticmethod
|
|
|
def _update_service_active() -> bool | None:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
["/usr/bin/systemctl", "show", "--property=ActiveState", "--value",
|
|
|
"virt-cloud-update.service"],
|
|
|
cwd="/",
|
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=5,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
|
return None
|
|
|
if result.returncode != 0:
|
|
|
return None
|
|
|
state = result.stdout.strip()
|
|
|
if state in {"active", "activating", "deactivating", "reloading"}:
|
|
|
return True
|
|
|
if state in {"inactive", "failed"}:
|
|
|
return False
|
|
|
return None
|
|
|
|
|
|
@staticmethod
|
|
|
def _stop_update_service() -> None:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
["/usr/bin/systemctl", "stop", "virt-cloud-update.service"],
|
|
|
cwd="/",
|
|
|
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
|
|
capture_output=True,
|
|
|
text=True,
|
|
|
timeout=30,
|
|
|
check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
|
raise UpdateServiceUncertain(
|
|
|
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503) from exc
|
|
|
if result.returncode != 0:
|
|
|
raise UpdateServiceUncertain(
|
|
|
"无法停止未确认的更新服务,请立即检查 systemd 状态", 503)
|
|
|
|
|
|
async def _reserve_operation(self, host_id, delegated, body, *, resource="vms", method="POST", target=""):
|
|
|
op_id = delegated["operation_id"]
|
|
|
# Even rejected signed provisioning requests have a durable outcome.
|
|
|
# A worker that loses this HTTP response can release its frozen balance
|
|
|
# only by querying this same operation, without replaying the request.
|
|
|
from .app import (
|
|
|
ActionBody, CreateCloudBody, CreateISOBody, FirewallPolicyBody,
|
|
|
)
|
|
|
from pydantic import ValidationError
|
|
|
create = resource == "vms"
|
|
|
value = {}
|
|
|
error = None
|
|
|
try:
|
|
|
value = json.loads(body) if body else {}
|
|
|
if not isinstance(value, dict):
|
|
|
raise ValueError("body must be an object")
|
|
|
if create:
|
|
|
(CreateCloudBody if value.get("source") == "cloud" else CreateISOBody).model_validate(value)
|
|
|
elif self._is_power_action_resource(resource):
|
|
|
# Persist only the reviewed action name. The original body
|
|
|
# may contain future/private fields and must never become a
|
|
|
# reconciliation instruction after a process restart.
|
|
|
value = ActionBody.model_validate(value).model_dump()
|
|
|
elif self._is_firewall_resource(resource):
|
|
|
value = FirewallPolicyBody.model_validate(value).model_dump()
|
|
|
from .libvirt_service import (
|
|
|
FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
firewall_policy_digest,
|
|
|
)
|
|
|
expected_digest = firewall_policy_digest(
|
|
|
value["enabled"], value["rules"])
|
|
|
if not hmac.compare_digest(
|
|
|
value["policy_digest"], expected_digest):
|
|
|
raise ValueError("firewall digest mismatch")
|
|
|
if value.get("guest_ssh_key_id") is not None or value.get("ssh_key_id") is not None:
|
|
|
error = "公钥必须由 PHP 校验用户归属后传递,不能使用旧公钥 ID"
|
|
|
except (ValueError, AttributeError, TypeError, ValidationError):
|
|
|
value = {}
|
|
|
error = "虚拟机参数校验失败"
|
|
|
record = {"operation_id": op_id, "host_id": host_id, "user_id": delegated["user_id"],
|
|
|
"request_hash": hmac.new(self.secret, method.encode() + b"\n" + target.encode() + b"\n" + (body or b""), hashlib.sha256).hexdigest(),
|
|
|
"vm_uuid": str(uuid.uuid4()) if create else str(uuid.UUID(resource.split("/")[1])),
|
|
|
"vm_name": value.get("name"), "kind": "create" if create else "mutation",
|
|
|
"actor_can_manage": bool(delegated.get("can_manage")),
|
|
|
"actor_is_admin": bool(delegated.get("is_admin")),
|
|
|
"resource": resource, "method": method, "status": "running", "result": None,
|
|
|
"safe_to_refund": False, "created_at": int(time.time())}
|
|
|
if create:
|
|
|
# New PHP-backed creates always receive the reserved UUID below.
|
|
|
# Keep this durable marker so cleanup reconciliation checks the
|
|
|
# UUID-derived resource candidates, while old in-flight records
|
|
|
# continue to use their historical display-name disk convention.
|
|
|
record["resource_naming"] = "uuid-v1"
|
|
|
if self._is_power_action_resource(resource) and not error:
|
|
|
record["target_action"] = value["action"]
|
|
|
if resource.endswith("/service-resume"):
|
|
|
# Store only the allowlisted target-state option. It is needed to
|
|
|
# replay the exact same convergence request after a lost response;
|
|
|
# never persist arbitrary request bodies in the operation table.
|
|
|
record["target_options"] = {
|
|
|
"start_if_suspended": value.get("start_if_suspended") is True,
|
|
|
}
|
|
|
if self._is_network_change_resource(resource):
|
|
|
# The child flips this durable checkpoint immediately before
|
|
|
# claiming the target IP. Until then, a crash or 5xx cannot make
|
|
|
# the IP transition ambiguous.
|
|
|
record["network_mutation_started"] = False
|
|
|
target_cidr = value.get("public_ip_cidr")
|
|
|
if isinstance(target_cidr, str):
|
|
|
# This is the only request field retained for reconciliation.
|
|
|
# Credentials or arbitrary mutation bodies are never persisted.
|
|
|
record["network_target_cidr"] = target_cidr
|
|
|
target_pool_key = value.get("public_pool_key")
|
|
|
if (isinstance(target_pool_key, str)
|
|
|
and re.fullmatch(r"[0-9a-f]{32}", target_pool_key)):
|
|
|
record["network_target_pool_key"] = target_pool_key
|
|
|
if method == "DELETE" and resource.startswith("vms/"):
|
|
|
query = dict(parse_qsl(urlsplit(target).query, keep_blank_values=True))
|
|
|
record["delete_disks"] = query.get("delete_disks", "").casefold() in {
|
|
|
"1", "true", "yes", "on",
|
|
|
}
|
|
|
if self._is_firewall_resource(resource) and not error:
|
|
|
record["firewall_target"] = {
|
|
|
"enabled": value["enabled"],
|
|
|
"revision": value["revision"],
|
|
|
"policy_digest": value["policy_digest"],
|
|
|
"rules": value["rules"],
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
}
|
|
|
if error:
|
|
|
# No service, task, VM or volume mutation has been invoked.
|
|
|
record.update(status="failed", prevalidation_failed=True,
|
|
|
error=error, safe_to_refund=create, finished_at=int(time.time()))
|
|
|
if self._is_firewall_resource(resource):
|
|
|
record["error_code"] = "firewall_rule_invalid"
|
|
|
fresh = await asyncio.to_thread(self.store.insert, "operation", op_id, json.dumps(record))
|
|
|
if fresh:
|
|
|
return record, True
|
|
|
previous = await self._operation(op_id, delegated, reconcile=False)
|
|
|
if previous["host_id"] != host_id or previous["user_id"] != delegated["user_id"] or previous["request_hash"] != record["request_hash"]:
|
|
|
raise EngineAuthError("相同开通操作编号的请求内容不一致", 409)
|
|
|
return previous, False
|
|
|
|
|
|
async def _operation(self, op_id, delegated, *, reconcile=True):
|
|
|
raw = await asyncio.to_thread(self.store.get, "operation", op_id)
|
|
|
if raw is None:
|
|
|
raise EngineAuthError("开通操作不存在", 404)
|
|
|
record = json.loads(raw)
|
|
|
if record["user_id"] != delegated["user_id"] and not delegated["is_admin"]:
|
|
|
raise EngineAuthError("开通操作不存在", 404)
|
|
|
# Never automatically create again. Recover a known UUID if libvirt
|
|
|
# committed while the response or process was lost.
|
|
|
if reconcile and record.get("kind") == "mutation" and record["status"] in {"running", "pending", "uncertain"}:
|
|
|
return await self._mutation_operation(record)
|
|
|
if (reconcile and record["status"] in {"running", "uncertain"}
|
|
|
and op_id not in self.active_operations
|
|
|
and (record["status"] == "uncertain" or int(time.time()) - record.get("created_at", 0) >= 120)):
|
|
|
await self.registry.get(record["host_id"])
|
|
|
components = self.registry.components[record["host_id"]]
|
|
|
profile = components.manager.current()
|
|
|
if profile and not profile.get("key_locked"):
|
|
|
try:
|
|
|
vm = await asyncio.to_thread(self._service(profile).get_vm, record["vm_uuid"])
|
|
|
if vm.get("state") != "running" or not vm.get("managed"):
|
|
|
return record
|
|
|
from .metrics import host_scope
|
|
|
await asyncio.to_thread(components.state.users.assign_vm,
|
|
|
host_scope(profile), record["vm_uuid"], record["user_id"])
|
|
|
record.update(status="succeeded", result=vm, safe_to_refund=False)
|
|
|
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
|
|
|
except Exception:
|
|
|
# A missing VM can be a transient connection failure or an
|
|
|
# operation still in progress. It is not permission to retry.
|
|
|
await self._check_absent(record)
|
|
|
await asyncio.to_thread(self.store.set, "operation", op_id, json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
async def _mutation_operation(self, record):
|
|
|
app = await self.registry.get(record["host_id"])
|
|
|
components = self.registry.components[record["host_id"]]
|
|
|
if (self._is_vm_delete_operation(record)
|
|
|
and record.get("cleanup_pending") in {
|
|
|
"public_ip_claim", "vm_delete",
|
|
|
}):
|
|
|
return await self._delete_cleanup_operation(record, components)
|
|
|
if self._is_firewall_resource(record.get("resource", "")):
|
|
|
return await self._firewall_operation(record, components)
|
|
|
if self._is_network_change_resource(record.get("resource", "")):
|
|
|
return await self._network_change_operation(record, components)
|
|
|
if self._is_power_action_resource(record.get("resource", "")):
|
|
|
return await self._power_action_operation(record, components)
|
|
|
target_method = self._service_target_method(record.get("resource", ""))
|
|
|
if target_method is not None:
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
return record
|
|
|
if (target_method == "service_suspend"
|
|
|
and not isinstance(record.get("service_was_running_hint"), bool)):
|
|
|
# Suspending without a durable pre-stop state can permanently
|
|
|
# lose whether a later paid renewal is allowed to restart the
|
|
|
# guest. Fail closed instead of replaying a destructive target
|
|
|
# after the database recovers.
|
|
|
record.update(
|
|
|
status="failed",
|
|
|
error="停机前运行状态未安全保存,已取消服务暂停",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
profile = components.manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return record
|
|
|
if (not record.get("actor_can_manage")
|
|
|
or (not record.get("actor_is_admin")
|
|
|
and not components.state.users.owns_vm(
|
|
|
self._host_scope(profile), record["vm_uuid"], record["user_id"]))):
|
|
|
record.update(status="failed", error="虚拟机不存在", finished_at=int(time.time()))
|
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
self.active_operations.add(record["operation_id"])
|
|
|
try:
|
|
|
async with self._target_lock(record["host_id"], record["vm_uuid"]):
|
|
|
options = record.get("target_options", {}) \
|
|
|
if target_method == "service_resume" else {}
|
|
|
if not isinstance(options, dict) or any(
|
|
|
key != "start_if_suspended" for key in options):
|
|
|
options = {}
|
|
|
result = await asyncio.to_thread(
|
|
|
getattr(self._service(profile), target_method), record["vm_uuid"],
|
|
|
**options)
|
|
|
if (target_method == "service_suspend"
|
|
|
and isinstance(record.get("service_was_running_hint"), bool)):
|
|
|
result["service_was_running"] = (
|
|
|
bool(result.get("service_was_running"))
|
|
|
or record["service_was_running_hint"])
|
|
|
except Exception as exc:
|
|
|
from .libvirt_service import VMError
|
|
|
if isinstance(exc, VMError) and exc.status < 500:
|
|
|
record.update(status="failed", error=str(exc), finished_at=int(time.time()))
|
|
|
else:
|
|
|
record.update(status="uncertain",
|
|
|
error="服务状态尚未确认,引擎将安全重试")
|
|
|
else:
|
|
|
record.update(status="succeeded", result=result, error=None,
|
|
|
finished_at=int(time.time()))
|
|
|
finally:
|
|
|
self.active_operations.discard(record["operation_id"])
|
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
if self._is_vm_delete_operation(record):
|
|
|
return await self._uncertain_delete_operation(record)
|
|
|
kind = (
|
|
|
"snapshot_job" if "/snapshots" in record["resource"]
|
|
|
else "reinstall_job" if record["resource"].endswith("/reinstall")
|
|
|
else "rescue_job" if "/rescue" in record["resource"]
|
|
|
else None
|
|
|
)
|
|
|
if kind is None:
|
|
|
# A reset/shutdown action cannot be safely replayed merely because
|
|
|
# its response disappeared. Once no executor can still own the
|
|
|
# operation, terminate it visibly instead of allowing a permanent
|
|
|
# running/uncertain row to defeat expiry enforcement.
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
scoped = components.store
|
|
|
job = None
|
|
|
job_id = record.get("job_id")
|
|
|
if job_id:
|
|
|
raw = await asyncio.to_thread(scoped.get, kind, job_id)
|
|
|
job = json.loads(raw) if raw else None
|
|
|
else:
|
|
|
rows = await asyncio.to_thread(scoped.list, kind)
|
|
|
for candidate_id, raw in rows.items():
|
|
|
candidate = json.loads(raw)
|
|
|
if candidate.get("operation_id") == record["operation_id"]:
|
|
|
job_id, job = candidate_id, candidate
|
|
|
break
|
|
|
if job is None:
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
if kind == "snapshot_job" and job_id:
|
|
|
state_getter = getattr(app.state, "snapshot_job_state", None)
|
|
|
if callable(state_getter):
|
|
|
live_job = await asyncio.to_thread(state_getter, job_id)
|
|
|
if (isinstance(live_job, dict)
|
|
|
and live_job.get("operation_id") == record["operation_id"]
|
|
|
and live_job.get("vm_id") == record["vm_uuid"]):
|
|
|
if live_job != job:
|
|
|
job = live_job
|
|
|
try:
|
|
|
await asyncio.to_thread(
|
|
|
scoped.set, kind, job_id,
|
|
|
json.dumps(job, ensure_ascii=False,
|
|
|
separators=(",", ":")),
|
|
|
)
|
|
|
except Exception:
|
|
|
# The in-memory terminal/uncertain state remains
|
|
|
# authoritative for this process and will be
|
|
|
# offered for persistence again on the next poll.
|
|
|
LOG.exception(
|
|
|
"Could not retry snapshot job persistence: job=%s",
|
|
|
job_id,
|
|
|
)
|
|
|
if job.get("owner") != f"php:{record['user_id']}" or job.get("vm_id") != record["vm_uuid"]:
|
|
|
return record
|
|
|
if kind == "snapshot_job" and job.get("status") == "uncertain":
|
|
|
reconciler = getattr(app.state, "reconcile_snapshot_job", None)
|
|
|
if callable(reconciler):
|
|
|
try:
|
|
|
async with self._target_lock(
|
|
|
record["host_id"], record["vm_uuid"]):
|
|
|
job = await asyncio.to_thread(reconciler, job_id)
|
|
|
except Exception:
|
|
|
# Reconciliation is evidence-only and never replays the
|
|
|
# original mutation. Temporary host/database failures keep
|
|
|
# the operation uncertain for a later poll.
|
|
|
LOG.exception(
|
|
|
"Snapshot reconciliation remains uncertain: op=%s",
|
|
|
record["operation_id"],
|
|
|
)
|
|
|
record["job_id"] = job_id
|
|
|
if kind == "snapshot_job":
|
|
|
if isinstance(job.get("result"), dict):
|
|
|
record["result"] = dict(job["result"])
|
|
|
else:
|
|
|
# Snapshot jobs persist private host evidence (for example
|
|
|
# deterministic volume identities) for safe reconciliation.
|
|
|
# Never copy the raw job into an operation response.
|
|
|
record["result"] = {
|
|
|
key: job[key]
|
|
|
for key in (
|
|
|
"job_id", "snapshot_key", "action", "consistency",
|
|
|
"status", "phase", "created_at", "started_at",
|
|
|
"finished_at",
|
|
|
)
|
|
|
if key in job
|
|
|
}
|
|
|
else:
|
|
|
record["result"] = {
|
|
|
key: value for key, value in job.items()
|
|
|
if key not in {"scope", "owner", "operation_id"}
|
|
|
}
|
|
|
record["result"]["job_id"] = job_id
|
|
|
status = job.get("status")
|
|
|
record["status"] = (
|
|
|
"succeeded" if status in {"completed", "succeeded"}
|
|
|
else "running" if status in {"queued", "running"}
|
|
|
else "uncertain" if status == "uncertain"
|
|
|
else "failed"
|
|
|
)
|
|
|
committed_reinstall = (
|
|
|
kind == "reinstall_job"
|
|
|
and _reinstall_committed(job)
|
|
|
)
|
|
|
if (committed_reinstall
|
|
|
and record["status"] in {"queued", "running", "succeeded", "failed"}):
|
|
|
# A progress marker is written after the replacement domain has
|
|
|
# started. A later job/result write may be missing, so do not keep
|
|
|
# exposing a committed reinstall as running or failed forever.
|
|
|
record["status"] = "uncertain"
|
|
|
record.pop("finished_at", None)
|
|
|
if committed_reinstall and job.get("error_code") not in REINSTALL_PUBLIC_ERROR_CODES:
|
|
|
record["error_code"] = (
|
|
|
"reinstall_cleanup_pending"
|
|
|
if _reinstall_cleanup_pending(job)
|
|
|
else "reinstall_result_uncertain"
|
|
|
)
|
|
|
if (("面板重启" in job.get("phase", "")
|
|
|
or "面板在任务执行期间重启" in job.get("error", ""))
|
|
|
and not committed_reinstall):
|
|
|
# The asynchronous job registry explicitly confirms that this job
|
|
|
# is no longer running. Replaying a reinstall/rescue mutation is
|
|
|
# unsafe, but leaving it uncertain forever would also grant free
|
|
|
# service by permanently blocking expiry enforcement.
|
|
|
record["status"] = "failed"
|
|
|
record["error"] = job.get("error") or "引擎重启导致任务中断,请核查虚拟机状态后重试"
|
|
|
if job.get("error"):
|
|
|
record["error"] = job["error"]
|
|
|
error_code = job.get("error_code")
|
|
|
if kind == "snapshot_job" and error_code in SNAPSHOT_PUBLIC_ERROR_CODES:
|
|
|
record["error_code"] = error_code
|
|
|
if kind == "reinstall_job" and error_code in REINSTALL_PUBLIC_ERROR_CODES:
|
|
|
record["error_code"] = error_code
|
|
|
if record["status"] in {"succeeded", "failed"}:
|
|
|
record["finished_at"] = int(job.get("finished_at") or time.time())
|
|
|
elif record["status"] == "uncertain":
|
|
|
record.pop("finished_at", None)
|
|
|
await asyncio.to_thread(self.store.set, "operation", record["operation_id"], json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
@staticmethod
|
|
|
def _validated_delete_cleanup_checkpoint(record: dict) -> dict:
|
|
|
"""Validate private identifiers before any post-delete cleanup."""
|
|
|
from .libvirt_service import SEED_VOLUME_RE, snapshot_volume_name
|
|
|
|
|
|
cleanup_pending = record.get("cleanup_pending")
|
|
|
if (cleanup_pending == "vm_delete"
|
|
|
and record.get("vm_delete_committed") is not True):
|
|
|
raise ValueError("delete commit boundary is not durable")
|
|
|
vm_id = record.get("vm_uuid")
|
|
|
try:
|
|
|
vm_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise ValueError("invalid delete cleanup VM") from exc
|
|
|
checkpoint = record.get("delete_cleanup_checkpoint")
|
|
|
if checkpoint is None and record.get("cleanup_pending") == "public_ip_claim":
|
|
|
# Compatibility for operations durably recorded by releases that
|
|
|
# only captured IP claims. Reaching that checkpoint already proved
|
|
|
# snapshot, rescue, ownership and credential cleanup completed.
|
|
|
checkpoint = {
|
|
|
"version": 1,
|
|
|
"vm_uuid": vm_id,
|
|
|
"snapshots": [],
|
|
|
"system_cleanup": False,
|
|
|
"legacy_ip_only": True,
|
|
|
}
|
|
|
allowed_checkpoint_keys = {
|
|
|
"version", "vm_uuid", "snapshots", "system_cleanup", "resources",
|
|
|
}
|
|
|
if cleanup_pending == "public_ip_claim":
|
|
|
allowed_checkpoint_keys.add("legacy_ip_only")
|
|
|
version = checkpoint.get("version") if isinstance(checkpoint, dict) else None
|
|
|
if (not isinstance(checkpoint, dict)
|
|
|
or set(checkpoint) - allowed_checkpoint_keys
|
|
|
or version not in {1, 2}
|
|
|
or checkpoint.get("vm_uuid") != vm_id
|
|
|
or not isinstance(checkpoint.get("system_cleanup"), bool)):
|
|
|
raise ValueError("invalid delete cleanup checkpoint")
|
|
|
resources = checkpoint.get("resources")
|
|
|
if version == 1 and resources is not None:
|
|
|
raise ValueError("invalid legacy delete resource checkpoint")
|
|
|
if version == 2:
|
|
|
if (not isinstance(resources, dict)
|
|
|
or set(resources) != {"seed_volume", "primary_volume"}):
|
|
|
raise ValueError("invalid delete resource checkpoint")
|
|
|
seed_volume = resources.get("seed_volume")
|
|
|
primary_volume = resources.get("primary_volume")
|
|
|
# Reinstall checkpoints carry the exact seed captured before
|
|
|
# undefine; reinstall seeds use a fresh UUID unrelated to vm_id.
|
|
|
if (seed_volume is not None
|
|
|
and (not isinstance(seed_volume, str)
|
|
|
or SEED_VOLUME_RE.fullmatch(seed_volume) is None)):
|
|
|
raise ValueError("invalid delete seed identity")
|
|
|
if (not isinstance(primary_volume, str)
|
|
|
or re.fullmatch(
|
|
|
r"[A-Za-z0-9][A-Za-z0-9._-]{0,62}\.qcow2",
|
|
|
primary_volume,
|
|
|
) is None):
|
|
|
raise ValueError("invalid delete primary disk identity")
|
|
|
snapshots = checkpoint.get("snapshots")
|
|
|
if (not isinstance(snapshots, list)
|
|
|
or len(snapshots) > DELETE_CLEANUP_SNAPSHOT_LIMIT):
|
|
|
raise ValueError("invalid delete snapshot checkpoint")
|
|
|
validated = []
|
|
|
seen = set()
|
|
|
for item in snapshots:
|
|
|
if not isinstance(item, dict) or set(item) != {
|
|
|
"snapshot_key", "volume_name"}:
|
|
|
raise ValueError("invalid delete snapshot entry")
|
|
|
snapshot_key = item.get("snapshot_key")
|
|
|
volume_name = item.get("volume_name")
|
|
|
if (not isinstance(snapshot_key, str)
|
|
|
or re.fullmatch(r"[0-9a-f]{32}", snapshot_key) is None
|
|
|
or snapshot_key in seen
|
|
|
or volume_name != snapshot_volume_name(vm_id, snapshot_key)):
|
|
|
raise ValueError("invalid deterministic snapshot identity")
|
|
|
seen.add(snapshot_key)
|
|
|
validated.append({
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"volume_name": volume_name,
|
|
|
})
|
|
|
keys = record.get("delete_ip_claim_keys")
|
|
|
if (not isinstance(keys, list) or len(keys) > 64
|
|
|
or len(set(keys)) != len(keys)
|
|
|
or any(not isinstance(key, str) or not key or len(key) > 160
|
|
|
for key in keys)):
|
|
|
raise ValueError("invalid delete IP claim checkpoint")
|
|
|
validated_checkpoint = {
|
|
|
**checkpoint,
|
|
|
"vm_uuid": vm_id,
|
|
|
"snapshots": validated,
|
|
|
"ip_claim_keys": list(keys),
|
|
|
}
|
|
|
if version == 2:
|
|
|
validated_checkpoint["resources"] = {
|
|
|
"seed_volume": resources.get("seed_volume"),
|
|
|
"primary_volume": resources["primary_volume"],
|
|
|
}
|
|
|
return validated_checkpoint
|
|
|
|
|
|
async def _delete_cleanup_operation(self, record, components):
|
|
|
"""Finish exact cleanup after VM deletion was durably proven.
|
|
|
|
|
|
This method never calls ``delete_vm``. Every destructive identifier is
|
|
|
captured before the original child request and validated again here.
|
|
|
Missing volumes/rows are accepted so a crash between cleanup steps is
|
|
|
safely idempotent.
|
|
|
"""
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
return record
|
|
|
try:
|
|
|
checkpoint = self._validated_delete_cleanup_checkpoint(record)
|
|
|
except (TypeError, ValueError):
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="删除任务缺少可信的清理检查点,必须人工核查",
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
profile = components.manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return record
|
|
|
vm_id = checkpoint["vm_uuid"]
|
|
|
scope = self._host_scope(profile)
|
|
|
try:
|
|
|
expected_snapshots = {
|
|
|
item["snapshot_key"]: item["volume_name"]
|
|
|
for item in checkpoint["snapshots"]
|
|
|
}
|
|
|
async with self._target_lock(record["host_id"], vm_id):
|
|
|
# Validate all locally persisted identities and claim
|
|
|
# ownership while holding the same per-VM lock as ordinary
|
|
|
# mutations, before deleting any remaining resource.
|
|
|
for key, raw in (await asyncio.to_thread(
|
|
|
components.store.list, "vm_snapshot")).items():
|
|
|
try:
|
|
|
snapshot = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise ValueError("damaged snapshot catalog") from exc
|
|
|
continue
|
|
|
if not isinstance(snapshot, dict):
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise ValueError("damaged snapshot catalog")
|
|
|
continue
|
|
|
if snapshot.get("vm_id") != vm_id:
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise ValueError("snapshot catalog VM changed")
|
|
|
continue
|
|
|
snapshot_key = snapshot.get("snapshot_key")
|
|
|
if (not isinstance(snapshot_key, str)
|
|
|
or key != f"{vm_id}:{snapshot_key}"
|
|
|
or expected_snapshots.get(snapshot_key)
|
|
|
!= snapshot.get("volume_name")):
|
|
|
raise ValueError(
|
|
|
"snapshot catalog changed after delete")
|
|
|
|
|
|
for key in checkpoint["ip_claim_keys"]:
|
|
|
raw = await asyncio.to_thread(
|
|
|
components.store.get, "ip_claim", key)
|
|
|
if raw is None:
|
|
|
continue
|
|
|
claim = json.loads(raw)
|
|
|
if (not isinstance(claim, dict)
|
|
|
or claim.get("vm_uuid") != vm_id):
|
|
|
raise ValueError("claim ownership changed")
|
|
|
|
|
|
service = self._service(profile)
|
|
|
resources = checkpoint.get("resources")
|
|
|
if resources is not None:
|
|
|
await asyncio.to_thread(
|
|
|
service.cleanup_deleted_vm_volumes, vm_id, resources)
|
|
|
for item in checkpoint["snapshots"]:
|
|
|
await asyncio.to_thread(
|
|
|
service.delete_snapshot_volume,
|
|
|
vm_id, item["snapshot_key"], item["volume_name"],
|
|
|
allow_missing_vm=True,
|
|
|
)
|
|
|
catalog_key = f"{vm_id}:{item['snapshot_key']}"
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "vm_snapshot", catalog_key)
|
|
|
if await asyncio.to_thread(
|
|
|
components.store.get,
|
|
|
"vm_snapshot", catalog_key) is not None:
|
|
|
raise RuntimeError("snapshot catalog delete not durable")
|
|
|
|
|
|
await asyncio.to_thread(
|
|
|
service.delete_firewall_filter, vm_id)
|
|
|
|
|
|
identity_key = f"{scope}.{vm_id}"
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "vm_firewall", identity_key)
|
|
|
if await asyncio.to_thread(
|
|
|
components.store.get,
|
|
|
"vm_firewall", identity_key) is not None:
|
|
|
raise RuntimeError("firewall record delete not durable")
|
|
|
if checkpoint["system_cleanup"]:
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "vm_rescue", identity_key)
|
|
|
if await asyncio.to_thread(
|
|
|
components.store.get,
|
|
|
"vm_rescue", identity_key) is not None:
|
|
|
raise RuntimeError("rescue state delete not durable")
|
|
|
|
|
|
await asyncio.to_thread(
|
|
|
components.state.users.delete_vm_owner, scope, vm_id)
|
|
|
owner_of = getattr(components.state.users, "owner_of_vm", None)
|
|
|
if not callable(owner_of):
|
|
|
raise RuntimeError("VM owner delete cannot be verified")
|
|
|
if await asyncio.to_thread(owner_of, scope, vm_id) is not None:
|
|
|
raise RuntimeError("VM owner delete not durable")
|
|
|
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "vm_credential", identity_key)
|
|
|
if await asyncio.to_thread(
|
|
|
components.store.get,
|
|
|
"vm_credential", identity_key) is not None:
|
|
|
raise RuntimeError("credential delete not durable")
|
|
|
|
|
|
for key in checkpoint["ip_claim_keys"]:
|
|
|
await asyncio.to_thread(
|
|
|
components.store.delete, "ip_claim", key)
|
|
|
if await asyncio.to_thread(
|
|
|
components.store.get, "ip_claim", key) is not None:
|
|
|
raise RuntimeError("claim delete not durable")
|
|
|
|
|
|
# Never silently leave an uncheckpointed claim for the deleted
|
|
|
# VM, and never delete one that was not captured before the VM
|
|
|
# mutation.
|
|
|
for raw in (await asyncio.to_thread(
|
|
|
components.store.list, "ip_claim")).values():
|
|
|
claim = json.loads(raw)
|
|
|
if (isinstance(claim, dict)
|
|
|
and claim.get("vm_uuid") == vm_id):
|
|
|
raise RuntimeError("uncaptured VM claim remains")
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"VM delete cleanup remains incomplete: op=%s uuid=%s",
|
|
|
record.get("operation_id"), vm_id,
|
|
|
)
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="虚拟机已删除,但快照或控制面清理仍未完成",
|
|
|
)
|
|
|
record.pop("finished_at", None)
|
|
|
else:
|
|
|
record.update(
|
|
|
status="succeeded",
|
|
|
result={
|
|
|
"uuid": vm_id,
|
|
|
"deleted": True,
|
|
|
"deleted_snapshots": [
|
|
|
item["snapshot_key"]
|
|
|
for item in checkpoint["snapshots"]
|
|
|
],
|
|
|
"public_ip_claims_released": True,
|
|
|
"firewall_cleaned": True,
|
|
|
},
|
|
|
error=None,
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
record.pop("error_code", None)
|
|
|
record.pop("cleanup_pending", None)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
async def _network_change_operation(self, record, _components):
|
|
|
"""Keep an ambiguous one-shot public-IP change fail-closed.
|
|
|
|
|
|
A single VM metadata read cannot prove that inactive/live XML, the
|
|
|
NoCloud seed and clean-traffic filter all converged. It also cannot
|
|
|
prove that a partially failed rollback restored every artifact.
|
|
|
Before the durable mutation checkpoint, a lost/5xx result is a failed
|
|
|
preflight and no target-IP claim exists. After that checkpoint, a
|
|
|
single metadata read cannot prove every artifact converged, so both
|
|
|
reservations remain unavailable until manual reconciliation.
|
|
|
"""
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
return record
|
|
|
if record.get("network_mutation_started") is False:
|
|
|
record.update(
|
|
|
status="failed",
|
|
|
error="网络变更尚未开始;前置核对被中断",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="网络变更结果不确定,已保留新旧公网 IP,必须人工核查",
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
async def _firewall_operation(self, record, components):
|
|
|
"""Resolve a lost firewall response from exact nwfilter evidence only."""
|
|
|
from .libvirt_service import (
|
|
|
SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS,
|
|
|
build_firewall_filter_xml,
|
|
|
firewall_policy_digest,
|
|
|
)
|
|
|
from .platform_network_policy import (
|
|
|
default_platform_policy_snapshot,
|
|
|
normalize_platform_policy_snapshot,
|
|
|
)
|
|
|
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
return record
|
|
|
target_record = record.get("firewall_target")
|
|
|
try:
|
|
|
from .app import FirewallPolicyBody
|
|
|
if not isinstance(target_record, dict):
|
|
|
raise ValueError("firewall target must be an object")
|
|
|
if not set(target_record).issubset({
|
|
|
"enabled", "revision", "policy_digest", "rules",
|
|
|
"filter_version"}):
|
|
|
raise ValueError("firewall target contains unknown fields")
|
|
|
target_filter_version = target_record.get("filter_version", 1)
|
|
|
if (isinstance(target_filter_version, bool)
|
|
|
or not isinstance(target_filter_version, int)
|
|
|
or target_filter_version
|
|
|
not in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS):
|
|
|
raise ValueError("unsupported firewall target compiler")
|
|
|
target = FirewallPolicyBody.model_validate({
|
|
|
key: target_record[key]
|
|
|
for key in ("enabled", "revision", "policy_digest", "rules")
|
|
|
}).model_dump()
|
|
|
except Exception:
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="防火墙操作缺少可信目标策略,必须人工核查",
|
|
|
error_code="firewall_result_uncertain",
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
profile = components.manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return record
|
|
|
if (not record.get("actor_can_manage")
|
|
|
or (not record.get("actor_is_admin")
|
|
|
and not components.state.users.owns_vm(
|
|
|
self._host_scope(profile), record["vm_uuid"],
|
|
|
record["user_id"]))):
|
|
|
record.update(
|
|
|
status="failed", error="虚拟机不存在",
|
|
|
error_code="firewall_apply_failed",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
try:
|
|
|
async with self._target_lock(record["host_id"], record["vm_uuid"]):
|
|
|
evidence = await asyncio.to_thread(
|
|
|
self._service(profile).reconcile_firewall_policy,
|
|
|
record["vm_uuid"],
|
|
|
enabled=target["enabled"], rules=target["rules"],
|
|
|
compiler_version=target_filter_version,
|
|
|
)
|
|
|
firewall_key = (
|
|
|
f"{self._host_scope(profile)}.{record['vm_uuid']}")
|
|
|
previous_raw = await asyncio.to_thread(
|
|
|
components.store.get, "vm_firewall", firewall_key)
|
|
|
previous = None
|
|
|
previous_valid = previous_raw is None
|
|
|
if previous_raw is not None:
|
|
|
try:
|
|
|
previous = json.loads(previous_raw)
|
|
|
previous_policy = FirewallPolicyBody.model_validate({
|
|
|
"enabled": previous["enabled"],
|
|
|
"revision": previous["revision"],
|
|
|
"policy_digest": previous["policy_digest"],
|
|
|
"rules": previous["rules"],
|
|
|
}).model_dump()
|
|
|
previous_filter_version = previous.get(
|
|
|
"filter_version", 1)
|
|
|
previous_record_version = previous.get("version")
|
|
|
previous_platform_policy = (
|
|
|
normalize_platform_policy_snapshot(
|
|
|
previous["platform_policy_snapshot"])
|
|
|
if previous_record_version == 2
|
|
|
else default_platform_policy_snapshot()
|
|
|
)
|
|
|
previous_filter_digest = build_firewall_filter_xml(
|
|
|
record["vm_uuid"],
|
|
|
previous_policy["enabled"],
|
|
|
previous_policy["rules"],
|
|
|
compiler_version=previous_filter_version,
|
|
|
platform_policy_snapshot=previous_platform_policy,
|
|
|
)[3]
|
|
|
previous_valid = (
|
|
|
previous_record_version in {1, 2}
|
|
|
and previous.get("vm_id") == record["vm_uuid"]
|
|
|
and not isinstance(previous_filter_version, bool)
|
|
|
and isinstance(previous_filter_version, int)
|
|
|
and previous_filter_version
|
|
|
in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS
|
|
|
and not (
|
|
|
previous_record_version == 1
|
|
|
and previous_filter_version >= 4)
|
|
|
and (
|
|
|
previous_record_version == 1
|
|
|
or hmac.compare_digest(
|
|
|
previous.get(
|
|
|
"platform_policy_digest", ""),
|
|
|
previous_platform_policy["digest"],
|
|
|
)
|
|
|
)
|
|
|
and re.fullmatch(
|
|
|
r"[0-9a-f]{64}",
|
|
|
previous.get("filter_digest", ""),
|
|
|
) is not None
|
|
|
and hmac.compare_digest(
|
|
|
previous_policy["policy_digest"],
|
|
|
firewall_policy_digest(
|
|
|
previous_policy["enabled"],
|
|
|
previous_policy["rules"],
|
|
|
),
|
|
|
)
|
|
|
and hmac.compare_digest(
|
|
|
previous["filter_digest"],
|
|
|
previous_filter_digest,
|
|
|
)
|
|
|
)
|
|
|
previous = {
|
|
|
**previous,
|
|
|
**previous_policy,
|
|
|
"filter_version": previous_filter_version,
|
|
|
"platform_policy_snapshot": previous_platform_policy,
|
|
|
"platform_policy_digest": previous_platform_policy["digest"],
|
|
|
}
|
|
|
except Exception:
|
|
|
previous_valid = False
|
|
|
observed_policy_digest = evidence.get("policy_digest")
|
|
|
digest_matches = (
|
|
|
isinstance(observed_policy_digest, str)
|
|
|
and hmac.compare_digest(
|
|
|
target["policy_digest"], observed_policy_digest)
|
|
|
)
|
|
|
target_can_commit = (
|
|
|
previous_raw is None
|
|
|
or (
|
|
|
previous_valid
|
|
|
and isinstance(previous, dict)
|
|
|
and hmac.compare_digest(
|
|
|
previous["platform_policy_digest"],
|
|
|
evidence.get("platform_policy_digest", ""),
|
|
|
)
|
|
|
and previous["filter_version"]
|
|
|
<= target_filter_version
|
|
|
and (
|
|
|
previous["revision"] < target["revision"]
|
|
|
or (
|
|
|
previous["revision"] == target["revision"]
|
|
|
and hmac.compare_digest(
|
|
|
previous["policy_digest"],
|
|
|
target["policy_digest"],
|
|
|
)
|
|
|
)
|
|
|
)
|
|
|
)
|
|
|
)
|
|
|
target_matches = (
|
|
|
evidence.get("matches") is True and digest_matches)
|
|
|
if target_matches and target_can_commit:
|
|
|
result = {
|
|
|
"enabled": target["enabled"],
|
|
|
"revision": target["revision"],
|
|
|
"policy_digest": evidence["policy_digest"],
|
|
|
"filter_digest": evidence["expected_filter_digest"],
|
|
|
"filter_version": target_filter_version,
|
|
|
"rules": evidence["rules"],
|
|
|
"sync_state": "active",
|
|
|
}
|
|
|
applied_record = {
|
|
|
"version": 2,
|
|
|
"filter_version": target_filter_version,
|
|
|
"vm_id": record["vm_uuid"],
|
|
|
**{key: result[key] for key in (
|
|
|
"enabled", "revision", "rules",
|
|
|
"policy_digest", "filter_digest",
|
|
|
)},
|
|
|
"platform_policy_snapshot": evidence[
|
|
|
"platform_policy_snapshot"],
|
|
|
"platform_policy_digest": evidence[
|
|
|
"platform_policy_digest"],
|
|
|
"applied_at": int(time.time()),
|
|
|
}
|
|
|
await asyncio.to_thread(
|
|
|
components.store.set, "vm_firewall", firewall_key,
|
|
|
json.dumps(
|
|
|
applied_record, ensure_ascii=False,
|
|
|
separators=(",", ":")),
|
|
|
)
|
|
|
else:
|
|
|
baseline_old = (
|
|
|
previous_raw is None
|
|
|
and evidence.get("coherent") is True
|
|
|
and evidence.get("filter_mode") == "baseline"
|
|
|
)
|
|
|
observed_filter_digest = evidence.get("filter_digest")
|
|
|
recorded_old = (
|
|
|
previous_valid
|
|
|
and isinstance(previous, dict)
|
|
|
and (
|
|
|
previous["revision"] < target["revision"]
|
|
|
or (
|
|
|
previous["revision"] == target["revision"]
|
|
|
and previous["filter_version"]
|
|
|
< target_filter_version
|
|
|
and hmac.compare_digest(
|
|
|
previous["policy_digest"],
|
|
|
target["policy_digest"],
|
|
|
)
|
|
|
)
|
|
|
)
|
|
|
and evidence.get("coherent") is True
|
|
|
and evidence.get("filter_mode") == "managed"
|
|
|
and isinstance(observed_filter_digest, str)
|
|
|
and hmac.compare_digest(
|
|
|
observed_filter_digest,
|
|
|
previous["filter_digest"],
|
|
|
)
|
|
|
)
|
|
|
# Only an intact baseline or the exact prior persisted
|
|
|
# filter proves that the attempted target did not commit.
|
|
|
# Any partial/mixed/unreadable evidence remains uncertain.
|
|
|
result = (
|
|
|
None
|
|
|
if not target_matches and (baseline_old or recorded_old)
|
|
|
else ...
|
|
|
)
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"Firewall reconciliation remains uncertain: op=%s",
|
|
|
record.get("operation_id"),
|
|
|
)
|
|
|
result = ...
|
|
|
if isinstance(result, dict):
|
|
|
record.update(
|
|
|
status="succeeded", result=result, error=None,
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
record.pop("error_code", None)
|
|
|
elif result is None:
|
|
|
record.update(
|
|
|
status="failed",
|
|
|
error="引擎证据确认目标防火墙策略未生效",
|
|
|
error_code="firewall_apply_failed",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
else:
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="防火墙生效证据不完整,必须继续核对原操作",
|
|
|
error_code="firewall_result_uncertain",
|
|
|
)
|
|
|
record.pop("finished_at", None)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
async def _uncertain_delete_operation(self, record):
|
|
|
"""Keep a lost destructive delete bound to its original operation.
|
|
|
|
|
|
A missing executor does not prove whether the domain definition and
|
|
|
every panel-managed volume were removed. Converting this state to a
|
|
|
terminal failure would let PHP issue a second DELETE with a new
|
|
|
operation identifier. Only the explicit claim-cleanup path above may
|
|
|
finish automatically after the child endpoint durably proved that VM
|
|
|
deletion itself completed.
|
|
|
"""
|
|
|
if record["operation_id"] in self.active_operations:
|
|
|
return record
|
|
|
record.pop("finished_at", None)
|
|
|
record.update(
|
|
|
status="uncertain",
|
|
|
error="删除结果不确定,必须继续核对原操作,不能重复删除",
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
async def _power_action_operation(self, record, components):
|
|
|
"""Reconcile a lost synchronous power request by observation only.
|
|
|
|
|
|
A power command is never replayed after its executor disappears. The
|
|
|
engine action itself verifies the postcondition before returning, and
|
|
|
this path covers the remaining transport/process-loss window by
|
|
|
reading the lightweight VM state endpoint. A transitional or
|
|
|
mismatched state remains visible, while reboot/reset stay uncertain
|
|
|
when the final state cannot prove that the edge-triggered command
|
|
|
committed.
|
|
|
"""
|
|
|
if (record["operation_id"] in self.active_operations
|
|
|
or int(time.time()) - int(record.get("created_at", 0)) < 120):
|
|
|
return record
|
|
|
action = record.get("target_action")
|
|
|
if action not in POWER_ACTIONS:
|
|
|
# Rows written before target_action was introduced retain the old
|
|
|
# no-replay behavior; do not infer an action from arbitrary body
|
|
|
# data during recovery.
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
if (record.get("actor_can_manage") is not True
|
|
|
and record.get("actor_is_admin") is not True):
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return record
|
|
|
if not record.get("actor_is_admin"):
|
|
|
try:
|
|
|
owns = await asyncio.to_thread(
|
|
|
components.state.users.owns_vm,
|
|
|
self._host_scope(profile), record["vm_uuid"],
|
|
|
record["user_id"],
|
|
|
)
|
|
|
except Exception:
|
|
|
return record
|
|
|
if not owns:
|
|
|
return await self._finish_orphan_mutation(record)
|
|
|
service = self._service(profile)
|
|
|
getter = getattr(service, "get_vm_state", None)
|
|
|
if not callable(getter):
|
|
|
# Compatibility with a short-lived older service implementation;
|
|
|
# get_vm still returns the same bounded state field, at a higher
|
|
|
# cost. Production VMService always has get_vm_state().
|
|
|
getter = getattr(service, "get_vm", None)
|
|
|
if not callable(getter):
|
|
|
return record
|
|
|
from .libvirt_service import VMError
|
|
|
try:
|
|
|
# Keep the observation, conclusion, and durable operation write in
|
|
|
# the same per-VM critical section as every other mutation. A new
|
|
|
# action must not race this recovery read and make an otherwise
|
|
|
# matching state stale immediately.
|
|
|
async with self._target_lock(record["host_id"], record["vm_uuid"]):
|
|
|
try:
|
|
|
observed = await asyncio.to_thread(
|
|
|
getter, record["vm_uuid"])
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, VMError) and exc.status == 404:
|
|
|
record.update(
|
|
|
status="failed", result=None,
|
|
|
error="虚拟机不存在",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record),
|
|
|
)
|
|
|
# A transport/read failure is not evidence of either
|
|
|
# outcome; leave the record available for a later
|
|
|
# read-only reconciliation.
|
|
|
return record
|
|
|
if not isinstance(observed, dict):
|
|
|
return record
|
|
|
if observed.get("uuid") != record["vm_uuid"]:
|
|
|
return record
|
|
|
state = observed.get("state")
|
|
|
if action in POWER_ACTION_EDGE_ACTIONS:
|
|
|
# reset always leaves the domain running, and a normal
|
|
|
# reboot usually does too. A reboot that leaves the
|
|
|
# domain shut off may be valid under on_reboot=destroy,
|
|
|
# but this durable row does not contain that XML policy.
|
|
|
# No observed state can prove either edge-triggered
|
|
|
# command committed, so keep it uncertain even when the
|
|
|
# domain is paused, crashed, or in transition.
|
|
|
record.update(
|
|
|
status="uncertain", result=None,
|
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
|
)
|
|
|
record.pop("finished_at", None)
|
|
|
elif state not in POWER_ACTION_TERMINAL_STATES[action]:
|
|
|
record.update(
|
|
|
status="failed", result=None,
|
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
else:
|
|
|
result = {
|
|
|
key: observed[key]
|
|
|
for key in ("uuid", "name", "state")
|
|
|
if key in observed
|
|
|
}
|
|
|
record.update(
|
|
|
status="succeeded", result=result, error=None,
|
|
|
finished_at=int(time.time()), safe_to_refund=False,
|
|
|
)
|
|
|
record.pop("error_code", None)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"],
|
|
|
json.dumps(record))
|
|
|
return record
|
|
|
except Exception:
|
|
|
# A lock/store failure is not evidence of the VM's power state;
|
|
|
# leave the durable uncertainty available for a later pass.
|
|
|
LOG.exception(
|
|
|
"Power action reconciliation remains unresolved: op=%s",
|
|
|
record.get("operation_id"),
|
|
|
)
|
|
|
return record
|
|
|
|
|
|
async def _finish_orphan_mutation(self, record):
|
|
|
if (record["operation_id"] in self.active_operations
|
|
|
or int(time.time()) - int(record.get("created_at", 0)) < 120):
|
|
|
return record
|
|
|
record.update(
|
|
|
status="failed",
|
|
|
error="任务执行器已不存在,不会自动重放;请核查虚拟机实际状态",
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", record["operation_id"], json.dumps(record))
|
|
|
return record
|
|
|
|
|
|
@staticmethod
|
|
|
def _service_target_method(resource: str) -> str | None:
|
|
|
match = re.fullmatch(
|
|
|
rf"vms/{VM_ID}/((?:service|network)-(?:suspend|resume))", resource)
|
|
|
if match is None:
|
|
|
return None
|
|
|
return {
|
|
|
"service-suspend": "service_suspend",
|
|
|
"service-resume": "service_resume",
|
|
|
"network-suspend": "network_suspend",
|
|
|
"network-resume": "network_resume",
|
|
|
}[match[1]]
|
|
|
|
|
|
@staticmethod
|
|
|
def _is_network_change_resource(resource: str) -> bool:
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/network", resource) is not None
|
|
|
|
|
|
@staticmethod
|
|
|
def _is_power_action_resource(resource: str) -> bool:
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/actions", resource) is not None
|
|
|
|
|
|
@staticmethod
|
|
|
def _is_firewall_resource(resource: str) -> bool:
|
|
|
return re.fullmatch(rf"vms/{VM_ID}/firewall", resource) is not None
|
|
|
|
|
|
@staticmethod
|
|
|
def _is_vm_delete_operation(operation: dict) -> bool:
|
|
|
return (operation.get("method") == "DELETE"
|
|
|
and re.fullmatch(
|
|
|
rf"vms/{VM_ID}", operation.get("resource", ""))
|
|
|
is not None)
|
|
|
|
|
|
@staticmethod
|
|
|
def _host_scope(profile):
|
|
|
from .metrics import host_scope
|
|
|
return host_scope(profile)
|
|
|
|
|
|
async def _capture_service_was_running_hint(self, host_id, operation, delegated):
|
|
|
# Run this inside the same per-VM lock as service_suspend. If the
|
|
|
# engine crashes after stopping QEMU but before saving the response, a
|
|
|
# replay otherwise sees only a shutoff guest and cannot know whether a
|
|
|
# later renewal is allowed to start it automatically.
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = components.manager.current()
|
|
|
owns = (profile is not None and not profile.get("key_locked")
|
|
|
and (delegated.get("is_admin") or components.state.users.owns_vm(
|
|
|
self._host_scope(profile), operation["vm_uuid"], delegated["user_id"])))
|
|
|
if not owns:
|
|
|
return
|
|
|
before = await asyncio.to_thread(
|
|
|
self._service(profile).get_vm, operation["vm_uuid"])
|
|
|
operation["service_was_running_hint"] = before.get("state") == "running"
|
|
|
# This write is a prerequisite for stopping the guest. Propagate any
|
|
|
# failure so the child endpoint is never called without a durable hint.
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", operation["operation_id"],
|
|
|
json.dumps(operation))
|
|
|
|
|
|
async def _capture_network_transition(self, host_id, operation):
|
|
|
"""Persist the source address before invoking the one-shot mutation."""
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = components.manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
raise EngineAuthError(
|
|
|
"宿主机未解锁,无法保存网络变更前状态", 409)
|
|
|
target = operation.get("network_target_cidr")
|
|
|
before = await asyncio.to_thread(
|
|
|
self._service(profile).get_vm, operation["vm_uuid"])
|
|
|
source = before.get("public_ip_cidr") \
|
|
|
if isinstance(before, dict) else None
|
|
|
if not isinstance(source, str) or not isinstance(target, str):
|
|
|
raise EngineAuthError(
|
|
|
"无法保存网络变更前后的公网 IP,已取消执行", 409)
|
|
|
operation["network_source_cidr"] = source
|
|
|
operation["network_mutation_started"] = False
|
|
|
source_pool_key = before.get("public_pool_key") \
|
|
|
if isinstance(before, dict) else None
|
|
|
if (isinstance(source_pool_key, str)
|
|
|
and (source_pool_key == "legacy-default"
|
|
|
or re.fullmatch(r"[0-9a-f]{32}", source_pool_key))):
|
|
|
operation["network_source_pool_key"] = source_pool_key
|
|
|
# This durable write is a prerequisite for calling the child endpoint.
|
|
|
# If it fails, the exception prevents libvirt from being touched.
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", operation["operation_id"],
|
|
|
json.dumps(operation))
|
|
|
|
|
|
async def _capture_delete_cleanup(self, host_id, operation, delegated):
|
|
|
"""Persist exact private cleanup identities before deleting the VM."""
|
|
|
from .libvirt_service import snapshot_volume_name
|
|
|
|
|
|
components = self.registry.components[host_id]
|
|
|
vm_id = str(uuid.UUID(operation["vm_uuid"]))
|
|
|
snapshots = []
|
|
|
for key, raw in (await asyncio.to_thread(
|
|
|
components.store.list, "vm_snapshot")).items():
|
|
|
try:
|
|
|
snapshot = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise EngineAuthError(
|
|
|
"虚拟机快照目录损坏,已取消删除", 409) from exc
|
|
|
continue
|
|
|
if not isinstance(snapshot, dict):
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise EngineAuthError(
|
|
|
"虚拟机快照目录损坏,已取消删除", 409)
|
|
|
continue
|
|
|
if snapshot.get("vm_id") != vm_id:
|
|
|
if key.startswith(vm_id + ":"):
|
|
|
raise EngineAuthError(
|
|
|
"虚拟机快照目录身份不一致,已取消删除", 409)
|
|
|
continue
|
|
|
snapshot_key = snapshot.get("snapshot_key")
|
|
|
volume_name = snapshot.get("volume_name")
|
|
|
if (not isinstance(snapshot_key, str)
|
|
|
or re.fullmatch(r"[0-9a-f]{32}", snapshot_key) is None
|
|
|
or key != f"{vm_id}:{snapshot_key}"
|
|
|
or volume_name != snapshot_volume_name(vm_id, snapshot_key)):
|
|
|
raise EngineAuthError(
|
|
|
"虚拟机快照目录身份不一致,已取消删除", 409)
|
|
|
snapshots.append({
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"volume_name": volume_name,
|
|
|
})
|
|
|
# Current policy retains at most two usable recovery points, but old
|
|
|
# confirmed-failed catalog rows may predate automatic pruning. Keep a
|
|
|
# bounded compatibility window so expiry deletion can safely clean
|
|
|
# those deterministic volumes instead of becoming permanently stuck.
|
|
|
if len(snapshots) > DELETE_CLEANUP_SNAPSHOT_LIMIT or len({
|
|
|
item["snapshot_key"] for item in snapshots}) != len(snapshots):
|
|
|
raise EngineAuthError("虚拟机快照目录数量或身份无效,已取消删除", 409)
|
|
|
keys = []
|
|
|
for key, raw in (await asyncio.to_thread(
|
|
|
components.store.list, "ip_claim")).items():
|
|
|
try:
|
|
|
claim = json.loads(raw)
|
|
|
except (TypeError, ValueError):
|
|
|
continue
|
|
|
if (isinstance(claim, dict)
|
|
|
and claim.get("vm_uuid") == operation["vm_uuid"]):
|
|
|
keys.append(key)
|
|
|
operation["delete_ip_claim_keys"] = sorted(keys)
|
|
|
profile = await asyncio.to_thread(components.manager.current)
|
|
|
resources = None
|
|
|
if (operation.get("delete_disks") is True
|
|
|
and profile is not None and not profile.get("key_locked")):
|
|
|
service = self._service(profile)
|
|
|
capture = getattr(service, "delete_vm_cleanup_manifest", None)
|
|
|
if callable(capture):
|
|
|
resources = await asyncio.to_thread(capture, vm_id)
|
|
|
if (not isinstance(resources, dict)
|
|
|
or resources.get("vm_uuid") != vm_id):
|
|
|
raise EngineAuthError("虚拟机删除检查点无效,已取消删除", 409)
|
|
|
resources = {
|
|
|
"seed_volume": resources.get("seed_volume"),
|
|
|
"primary_volume": resources.get("primary_volume"),
|
|
|
}
|
|
|
checkpoint = {
|
|
|
"version": 2 if resources is not None else 1,
|
|
|
"vm_uuid": vm_id,
|
|
|
"snapshots": sorted(
|
|
|
snapshots, key=lambda item: item["snapshot_key"]),
|
|
|
"system_cleanup": delegated.get("system_cleanup") is True,
|
|
|
}
|
|
|
if resources is not None:
|
|
|
checkpoint["resources"] = resources
|
|
|
operation["delete_cleanup_checkpoint"] = checkpoint
|
|
|
# The complete private checkpoint is durable before libvirt or storage
|
|
|
# deletion begins. A write failure aborts the child call rather than
|
|
|
# losing exact cleanup proof.
|
|
|
await asyncio.to_thread(
|
|
|
self.store.set, "operation", operation["operation_id"],
|
|
|
json.dumps(operation))
|
|
|
|
|
|
async def _check_absent(self, operation):
|
|
|
operation["safe_to_refund"] = False
|
|
|
components = self.registry.components.get(operation["host_id"])
|
|
|
profile = components.manager.current() if components else None
|
|
|
if profile and not profile.get("key_locked"):
|
|
|
try:
|
|
|
absent = await asyncio.to_thread(self._service(profile).provisioning_absent,
|
|
|
operation["vm_uuid"], operation["vm_name"],
|
|
|
uuid_resources=operation.get("resource_naming") == "uuid-v1",
|
|
|
legacy_name_collision=(
|
|
|
operation.get("resource_naming") is None
|
|
|
and operation.get("error") == "虚拟机名称已存在"
|
|
|
))
|
|
|
if absent:
|
|
|
await asyncio.to_thread(self._cleanup_absent_records, components,
|
|
|
profile, operation["vm_uuid"])
|
|
|
operation.update(status="failed", safe_to_refund=True)
|
|
|
except Exception:
|
|
|
pass
|
|
|
|
|
|
@staticmethod
|
|
|
def _cleanup_absent_records(components, profile, vm_id):
|
|
|
from .metrics import host_scope
|
|
|
scope = host_scope(profile)
|
|
|
components.state.users.delete_vm_owner(scope, vm_id)
|
|
|
components.store.delete("vm_credential", f"{scope}.{vm_id}")
|
|
|
prefix = scope + "."
|
|
|
for key, raw in components.store.list("ip_claim").items():
|
|
|
if key.startswith(prefix) and json.loads(raw).get("vm_uuid") == vm_id:
|
|
|
components.store.delete("ip_claim", key)
|
|
|
|
|
|
def _public_operation(self, record):
|
|
|
result = {key: record[key] for key in (
|
|
|
"operation_id", "status", "result", "error", "safe_to_refund",
|
|
|
"service_was_running_hint", "created_at", "finished_at")
|
|
|
if key in record}
|
|
|
if ("/snapshots" in record.get("resource", "")
|
|
|
and record.get("error_code") in SNAPSHOT_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = record["error_code"]
|
|
|
if (record.get("resource", "").endswith("/reinstall")
|
|
|
and record.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = record["error_code"]
|
|
|
if (self._is_firewall_resource(record.get("resource", ""))
|
|
|
and record.get("error_code") in FIREWALL_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = record["error_code"]
|
|
|
if (self._is_vm_delete_operation(record)
|
|
|
and record.get("error_code") in DELETE_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = record["error_code"]
|
|
|
if (self._is_network_change_resource(record.get("resource", ""))
|
|
|
and record.get("network_mutation_started") is False
|
|
|
and record.get("error_code") in FIREWALL_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = record["error_code"]
|
|
|
# This is deliberately live process state, not a persisted claim. PHP
|
|
|
# must not close a stale policy job while its original executor can
|
|
|
# still mutate libvirt. It retries after the executor becomes terminal.
|
|
|
result["executor_active"] = record.get("operation_id") in self.active_operations
|
|
|
return result
|
|
|
|
|
|
@staticmethod
|
|
|
def _service(profile):
|
|
|
from .libvirt_service import VMService
|
|
|
return VMService(profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"],
|
|
|
profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""))
|
|
|
|
|
|
async def _ticket(self, host_id, vm_id, delegated, send):
|
|
|
if not delegated.get("session_id"):
|
|
|
raise EngineAuthError("控制台需要有效的网站会话", 403)
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = components.manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
raise EngineAuthError("宿主机尚未就绪", 409)
|
|
|
from .metrics import host_scope
|
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
|
|
|
raise EngineAuthError("虚拟机不存在", 404)
|
|
|
await asyncio.to_thread(self._service(profile).console_port, vm_id)
|
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
|
raise EngineAuthError("网站会话已失效", 403)
|
|
|
ticket = secrets.token_urlsafe(32)
|
|
|
key = hashlib.sha256(ticket.encode("ascii")).hexdigest()
|
|
|
record = {"host_id": host_id, "vm_id": vm_id, "actor": delegated, "expires_at": int(time.time()) + 60}
|
|
|
await asyncio.to_thread(self.store.insert, "console_ticket", key, json.dumps(record))
|
|
|
await json_response(send, 200, {"ticket": ticket, "expires_in": 60, "websocket_path": "/engine-console"})
|
|
|
|
|
|
async def _session_active(self, delegated, host_id, vm_id):
|
|
|
if self.introspection is not None:
|
|
|
return await self.introspection(delegated, host_id, vm_id)
|
|
|
return await asyncio.to_thread(self._introspect, delegated, host_id, vm_id)
|
|
|
|
|
|
def _introspect(self, delegated, host_id, vm_id):
|
|
|
url = self.config.get("php_introspection_url", "")
|
|
|
parsed = urlsplit(url)
|
|
|
if (parsed.scheme not in {"http", "https"} or parsed.hostname not in {"127.0.0.1", "::1", "localhost"}
|
|
|
or parsed.username or parsed.password or parsed.fragment):
|
|
|
return False
|
|
|
body = json.dumps({"session_id": delegated["session_id"], "user_id": delegated["user_id"],
|
|
|
"host_id": host_id, "vm_uuid": vm_id}, separators=(",", ":")).encode()
|
|
|
secret = decode_secret(self.config.get("introspection_secret", self.config["service_secret"]))
|
|
|
headers = signed_headers(secret, "POST", parsed.path + ("?" + parsed.query if parsed.query else ""), body, delegated)
|
|
|
headers["Content-Type"] = "application/json"
|
|
|
if self.config.get("php_introspection_host"):
|
|
|
headers["Host"] = self.config["php_introspection_host"]
|
|
|
try:
|
|
|
opener = build_opener(ProxyHandler({}), NoRedirects())
|
|
|
with opener.open(URLRequest(url, body, headers, method="POST"), timeout=5) as response:
|
|
|
raw = response.read(8193)
|
|
|
return response.status == 200 and len(raw) <= 8192 and json.loads(raw).get("active") is True
|
|
|
except Exception:
|
|
|
return False
|
|
|
|
|
|
async def _console(self, scope, receive, send):
|
|
|
from starlette.websockets import WebSocket
|
|
|
socket = WebSocket(scope, receive=receive, send=send)
|
|
|
if scope["path"] != "/engine-console" or scope.get("scheme") != "wss":
|
|
|
await socket.close(code=4403)
|
|
|
return
|
|
|
from .app import _origin_matches_host
|
|
|
if not _origin_matches_host(socket.headers.get("origin"), socket.headers.get("host")):
|
|
|
await socket.close(code=4403)
|
|
|
return
|
|
|
protocols = [value for value in scope.get("subprotocols", []) if value.startswith("cloud-ticket.")]
|
|
|
if len(protocols) != 1 or not re.fullmatch(r"cloud-ticket\.[A-Za-z0-9_-]{43}", protocols[0]):
|
|
|
await socket.close(code=4401)
|
|
|
return
|
|
|
key = hashlib.sha256(protocols[0][13:].encode("ascii")).hexdigest()
|
|
|
raw = await asyncio.to_thread(self.store.get, "console_ticket", key)
|
|
|
if raw is None:
|
|
|
await socket.close(code=4401)
|
|
|
return
|
|
|
record = json.loads(raw)
|
|
|
if record["expires_at"] < int(time.time()) or not await asyncio.to_thread(self.store.insert, "ticket_used", key, "1"):
|
|
|
await socket.close(code=4401)
|
|
|
return
|
|
|
await asyncio.to_thread(self.store.delete, "console_ticket", key)
|
|
|
delegated, host_id, vm_id = record["actor"], record["host_id"], record["vm_id"]
|
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
|
await socket.close(code=4403)
|
|
|
return
|
|
|
token = TRUSTED_ACTOR.set(delegated)
|
|
|
relay = watcher = None
|
|
|
try:
|
|
|
await self.registry.get(host_id)
|
|
|
components = self.registry.components[host_id]
|
|
|
profile = components.manager.current()
|
|
|
from .metrics import host_scope
|
|
|
from .console import relay_console
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
await socket.close(code=4409)
|
|
|
return
|
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(profile), vm_id, delegated["user_id"]):
|
|
|
await socket.close(code=4403)
|
|
|
return
|
|
|
port = await asyncio.to_thread(self._service(profile).console_port, vm_id)
|
|
|
await socket.accept(subprotocol=protocols[0])
|
|
|
async def watch():
|
|
|
while True:
|
|
|
await asyncio.sleep(self.console_check_interval)
|
|
|
if not await self._session_active(delegated, host_id, vm_id):
|
|
|
return
|
|
|
current = components.manager.current()
|
|
|
if current is None or current.get("key_locked"):
|
|
|
return
|
|
|
if not delegated["is_admin"] and not components.state.users.owns_vm(host_scope(current), vm_id, delegated["user_id"]):
|
|
|
return
|
|
|
relay = asyncio.create_task(relay_console(socket, port=port,
|
|
|
ssh_target=profile["ssh_target"], ssh_binary="ssh",
|
|
|
known_hosts_file=profile["ssh_known_hosts_file"], ssh_config_file=profile["ssh_config_file"]))
|
|
|
watcher = asyncio.create_task(watch())
|
|
|
await asyncio.wait({relay, watcher}, return_when=asyncio.FIRST_COMPLETED)
|
|
|
except Exception:
|
|
|
LOG.warning("Engine console closed after transport/authorization failure")
|
|
|
finally:
|
|
|
for task in (relay, watcher):
|
|
|
if task is not None:
|
|
|
task.cancel()
|
|
|
await asyncio.gather(*(task for task in (relay, watcher) if task is not None), return_exceptions=True)
|
|
|
with suppress(Exception):
|
|
|
await socket.close(code=4403)
|
|
|
TRUSTED_ACTOR.reset(token)
|
|
|
|
|
|
async def _lifespan(self, receive, send):
|
|
|
while True:
|
|
|
message = await receive()
|
|
|
if message["type"] == "lifespan.startup":
|
|
|
try:
|
|
|
await self.registry.start()
|
|
|
# The supervisor is fed only by durable tombstones created
|
|
|
# by an explicit delete. It never makes reads or unrelated
|
|
|
# writes perform GC, pauses during online update, and keeps
|
|
|
# retrying offline/locked hosts after they recover.
|
|
|
self.cache_cleanup_event = asyncio.Event()
|
|
|
self.cache_cleanup_supervisor = asyncio.create_task(
|
|
|
self._cache_cleanup_loop())
|
|
|
self.cache_cleanup_event.set()
|
|
|
await send({"type": "lifespan.startup.complete"})
|
|
|
except Exception:
|
|
|
LOG.exception("Engine startup failed")
|
|
|
await send({"type": "lifespan.startup.failed", "message": "引擎初始化失败"})
|
|
|
return
|
|
|
elif message["type"] == "lifespan.shutdown":
|
|
|
supervisor = self.cache_cleanup_supervisor
|
|
|
if supervisor is not None:
|
|
|
supervisor.cancel()
|
|
|
await asyncio.gather(supervisor, return_exceptions=True)
|
|
|
self.cache_cleanup_supervisor = None
|
|
|
self.cache_cleanup_event = None
|
|
|
# A task awaiting asyncio.to_thread cannot stop the underlying
|
|
|
# libvirt call when cancelled. Drain these bounded cleanup
|
|
|
# tasks before closing host services so no hidden mutation can
|
|
|
# outlive the engine lifespan.
|
|
|
await asyncio.gather(
|
|
|
*tuple(self.maintenance_tasks), return_exceptions=True)
|
|
|
await self.registry.close()
|
|
|
await send({"type": "lifespan.shutdown.complete"})
|
|
|
return
|
|
|
|
|
|
|
|
|
def create_engine():
|
|
|
default = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
|
|
|
config = private_config(Path(os.environ.get("CLOUD_ENGINE_CONFIG", str(default))))
|
|
|
secret = decode_secret(config["service_secret"])
|
|
|
del secret
|
|
|
key = master_key(Path(config["master_key_file"]))
|
|
|
store = MainStore(config["main_database"], key)
|
|
|
logs = LogStore(config["log_database"], key)
|
|
|
return EngineApplication(config, store, logs)
|