"""Browser setup, authenticated API, and VM console for one remote host.""" from __future__ import annotations import asyncio import hashlib import hmac import ipaddress import json import logging import os import re import secrets import subprocess import tempfile import threading import time import uuid import xml.etree.ElementTree as ET from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Any, Callable, Literal from urllib.parse import urlsplit from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Query, Request, WebSocket from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator from .auth import DatabaseSessionStore, LoginLimiter, Session, SessionStore, verify_password from .cloud_init import ( _validate_guest_auth, _validate_ssh_public_key, generate_root_password, hash_root_password, validate_cloud_network, validate_guest_hostname, ) from .cloud_templates import ( TemplateError, TemplateStore, download_verified_image, image_metadata_for, template_id_for, template_scope, template_volume_name, validate_template_name, ) from .config import Settings from .console import ConsoleRelayError, relay_console from .database import DatabaseError, make_components from .database_bootstrap import DatabaseBootstrap, DatabaseSetupError from .libvirt_service import ( FIREWALL_FILTER_COMPILER_VERSION, REINSTALL_CLEANUP_PENDING_ERROR_CODE, REINSTALL_RESULT_UNCERTAIN_ERROR_CODE, SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS, SNAPSHOT_KEY_RE, VMError, VMService, VMSnapshotUncertainError, build_firewall_filter_xml, firewall_policy_digest, snapshot_volume_name, ) from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollector, host_scope, query_interval_seconds) from .panel_state import PanelState, SetupError from .platform_network_policy import ( default_platform_policy_snapshot, normalize_platform_policy_snapshot, ) from .remote_host import ( MAX_PUBLIC_NETWORK_POOLS, PUBLIC_POOL_KEY, HostProfile, RemoteHostError, RemoteHostManager, validate_public_network_pool, validate_public_network_pools, ) from .rescue_templates import ( MAX_RESCUE_ISO_BYTES, RescueTemplateStore, rescue_scope, rescue_volume_name, verify_bootable_iso, ) from .ssh_keys import SSHKeyError from .smtp_service import SMTPError, SMTPSettingsStore, send_verification_code from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater from .user_isos import ( UserISOFiles, UserISOStore, download_user_iso, user_iso_scope, user_iso_volume_name, validate_user_iso_name, validate_user_iso_url, ) from .users import UserError, normalize_email from .engine_context import EngineComponents, TRUSTED_ACTOR, actor, actor_account, actor_session COOKIE_NAME = "virsh_panel_session" CONSOLE_ACCESS_CHECK_SECONDS = 15 LOG = logging.getLogger(__name__) STARTUP_LOG = logging.getLogger("uvicorn.error") SNAPSHOT_PUBLIC_ERROR_CODES = { "snapshot_guest_agent_unavailable", "live_snapshot_unsupported", "snapshot_pool_unsupported", "snapshot_result_uncertain", } REINSTALL_PUBLIC_ERROR_CODES = { REINSTALL_CLEANUP_PENDING_ERROR_CODE, REINSTALL_RESULT_UNCERTAIN_ERROR_CODE, } # These phases are emitted only after the replacement definition has been # created or the old-volume cleanup boundary has been crossed. Keep this # compatibility list for rows written by releases that did not persist the # explicit ``reinstall_committed`` marker before a process restart. REINSTALL_COMMITTED_PHASES = frozenset({ "cleaning_up", "cleanup_pending", "result_uncertain", "completed", "重装已完成,但凭据或任务记录保存失败", "重装完成,但旧系统盘清理待核对", "重装结果待核对", }) def reinstall_phase_committed(value: Any) -> bool: return isinstance(value, str) and value.strip() in REINSTALL_COMMITTED_PHASES def reinstall_cleanup_pending(job: dict[str, Any]) -> bool: phase = job.get("phase") if isinstance(phase, str): phase = phase.strip() error_code = job.get("error_code") return ( error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE or job.get("cleanup_pending") is True or phase in {"cleanup_pending", "重装完成,但旧系统盘清理待核对"} or (isinstance(job.get("warning"), str) and job["warning"].strip().startswith("旧存储卷")) ) def reinstall_post_commit(job: dict[str, Any]) -> bool: error_code = job.get("error_code") return ( job.get("reinstall_committed") is True or error_code in REINSTALL_PUBLIC_ERROR_CODES or reinstall_phase_committed(job.get("phase")) or reinstall_cleanup_pending(job) or (isinstance(job.get("warning"), str) and bool(job["warning"].strip())) ) class PublicIpClaimCleanupPending(VMError): """The VM mutation completed but its engine IP claim did not.""" class VMDeleteSnapshotCleanupPending(VMError): """The VM and its primary disks are gone but snapshot GC is incomplete.""" def __init__(self, message: str) -> None: super().__init__( message, 503, error_code="vm_delete_snapshot_cleanup_pending", ) class VMDeleteFirewallCleanupPending(VMError): """The VM is gone but its deterministic nwfilter still needs cleanup.""" def __init__(self, message: str) -> None: super().__init__( message, 503, error_code="vm_delete_firewall_cleanup_pending", ) class SetupBody(BaseModel): model_config = ConfigDict(extra="forbid") token: str = Field(min_length=1, max_length=256) username: str = Field(min_length=3, max_length=32) password: str = Field(min_length=12, max_length=1024) class DatabaseSetupBody(BaseModel): model_config = ConfigDict(extra="forbid") host: str = Field(min_length=1, max_length=253) port: int = Field(ge=1, le=65535) database: str = Field(min_length=1, max_length=64) username: str = Field(min_length=1, max_length=128) password: SecretStr setup_token: str = Field(min_length=1, max_length=256) class UpdateSettingsBody(BaseModel): model_config = ConfigDict(extra="forbid") repository_url: str = Field(default=REPOSITORY_URL, max_length=2048) branch: str = Field(default=BRANCH, max_length=128) access_token: SecretStr | None = None class UpdateApplyBody(BaseModel): model_config = ConfigDict(extra="forbid") expected_current_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") expected_remote_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") class LoginBody(BaseModel): model_config = ConfigDict(extra="forbid") email: str | None = Field(default=None, min_length=3, max_length=254) # Only the injected, database-free test harness may use the historical # username login. Production explicitly rejects this field below. username: str | None = Field(default=None, min_length=1, max_length=64) password: str = Field(min_length=1, max_length=1024) class RegistrationCodeBody(BaseModel): model_config = ConfigDict(extra="forbid") email: str = Field(min_length=3, max_length=254) class RegistrationBody(RegistrationCodeBody): username: str = Field(min_length=3, max_length=32) password: str = Field(min_length=12, max_length=1024) code: str = Field(pattern=r"^[0-9]{6}$") class SMTPSettingsBody(BaseModel): model_config = ConfigDict(extra="forbid") host: str = Field(min_length=1, max_length=253) port: int = Field(ge=1, le=65535) tls_mode: Literal["ssl", "starttls", "none"] username: str = Field(default="", max_length=256) password: SecretStr | None = None from_address: str = Field(min_length=3, max_length=254) class UnlockBody(BaseModel): model_config = ConfigDict(extra="forbid") passphrase: str = Field(min_length=1, max_length=1024) class HostScanBody(BaseModel): model_config = ConfigDict(extra="forbid") host: str = Field(min_length=1, max_length=253) port: int = Field(ge=1, le=65535) class HostBody(HostScanBody): private_key: str = Field(default="", max_length=65536) private_key_passphrase: str = Field(default="", max_length=1024) host_key: str = Field(min_length=1, max_length=4096) confirmed_fingerprint: str = Field(min_length=1, max_length=256) storage_pool: str = Field(default="default", min_length=1, max_length=4096) iso_pool: str = Field(default="iso", min_length=1, max_length=4096) rescue_iso_pool: str = Field(default="", max_length=4096) default_network: str = Field(default="default", min_length=1, max_length=128) region: str = Field(default="", max_length=64, pattern=r"^[^\x00-\x1f\x7f]*$") public_pool_cidr: str = Field(default="", max_length=18) public_gateway: str = Field(default="", max_length=15) public_dns_servers: list[str] = Field(default_factory=list, max_length=4) public_reserved_ips: list[str] = Field(default_factory=list, max_length=254) public_vm_prefix: int | None = Field(default=None, ge=24, le=30) def profile(self) -> HostProfile: return HostProfile( host=self.host, port=self.port, private_key=self.private_key, host_key=self.host_key, private_key_passphrase=self.private_key_passphrase, storage_pool=self.storage_pool, iso_pool=self.iso_pool, rescue_iso_pool=self.rescue_iso_pool, default_network=self.default_network, region=self.region, public_pool_cidr=self.public_pool_cidr, public_gateway=self.public_gateway, public_dns_servers=self.public_dns_servers, public_reserved_ips=self.public_reserved_ips, public_vm_prefix=self.public_vm_prefix, ) class PublicNetworkPoolBody(BaseModel): model_config = ConfigDict(extra="forbid") revision: int = Field(ge=1, le=(1 << 63) - 1) state: Literal["active", "draining", "disabled"] = "active" network_selector: str = Field(min_length=1, max_length=128) cidr: str = Field(min_length=7, max_length=18) gateway: str = Field(min_length=7, max_length=15) dns_servers: list[str] = Field(min_length=1, max_length=4) reserved_ips: list[str] = Field(default_factory=list, max_length=254) guest_prefix_length: int = Field(ge=24, le=30) class CreateISOBody(BaseModel): model_config = ConfigDict(extra="forbid") source: Literal["iso"] = "iso" name: str = Field(min_length=1, max_length=63) vcpu: int = Field(ge=1, le=64) memory_mib: int = Field(ge=512, le=262144) disk_gib: int = Field(ge=1, le=4096) bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000) network: str = Field(min_length=1, max_length=128) iso: str | None = Field(default=None, min_length=1, max_length=255) user_iso_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{32}$") @model_validator(mode="after") def exactly_one_iso_source(self): if (self.iso is None) == (self.user_iso_id is None): raise ValueError("iso 与 user_iso_id 必须且只能选择一个") return self class CreateCloudBody(BaseModel): model_config = ConfigDict(extra="forbid") source: Literal["cloud"] name: str = Field(min_length=1, max_length=63) vcpu: int = Field(ge=1, le=64) memory_mib: int = Field(ge=512, le=262144) disk_gib: int = Field(ge=1, le=4096) bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000) network: str = Field(min_length=1, max_length=128) template_id: str = Field(min_length=1, max_length=80) public_pool_key: str = Field(default="", max_length=32, pattern=r"^(?:|[0-9a-f]{32})$") public_ip_cidr: str = Field(default="", max_length=64) gateway: str = Field(default="", max_length=64) dns_servers: list[str] = Field(default_factory=list, max_length=4) guest_ssh_public_key: str = Field(default="", max_length=16384) guest_ssh_key_id: int | None = Field(default=None, ge=1) root_password: SecretStr | None = None platform_network_policy_snapshot: dict[str, Any] = Field( default_factory=default_platform_policy_snapshot) @model_validator(mode="after") def validate_platform_network_policy(self): try: self.platform_network_policy_snapshot = ( normalize_platform_policy_snapshot( self.platform_network_policy_snapshot)) except ValueError as exc: raise ValueError("平台网络策略快照无效") from exc return self class ChangeNetworkBody(BaseModel): model_config = ConfigDict(extra="forbid") public_ip_cidr: str = Field(min_length=7, max_length=64) public_pool_key: str = Field(default="", max_length=32, pattern=r"^(?:|[0-9a-f]{32})$") confirm_restart: bool = False class VMSettingsBody(BaseModel): model_config = ConfigDict(extra="forbid") # Every field is optional so an administrator can change one setting at a # time. The validator below rejects an empty patch; omitted credentials # are preserved from the existing cloud-init seed. hostname: str | None = Field(default=None, max_length=253) root_password: SecretStr | None = Field(default=None, min_length=12, max_length=128) ssh_public_key: str | None = Field(default=None, max_length=16384) confirm_restart: bool = False @model_validator(mode="after") def require_setting(self): if self.hostname is None and self.root_password is None and self.ssh_public_key is None: raise ValueError("至少需要修改一项虚拟机设置") return self class SSHKeyBody(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(min_length=1, max_length=80) public_key: str = Field(min_length=1, max_length=16384) class ImportTemplateBody(BaseModel): model_config = ConfigDict(extra="forbid") url: str = Field(min_length=16, max_length=2048) sha256: str = Field(default="", max_length=64) name: str = Field(min_length=1, max_length=64) class ImportUserISOBody(BaseModel): model_config = ConfigDict(extra="forbid") url: str = Field(min_length=1, max_length=2048) sha256: str = Field(default="", max_length=64) name: str = Field(min_length=1, max_length=64) class RenameTemplateBody(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(min_length=1, max_length=64) class DeleteTemplateBody(BaseModel): model_config = ConfigDict(extra="forbid") created_at: int = Field(ge=1) class ReinstallBody(BaseModel): model_config = ConfigDict(extra="forbid") template_id: str = Field(min_length=1, max_length=80) ssh_public_key: str = Field(default="", max_length=16384) ssh_key_id: int | None = Field(default=None, ge=1) root_password: SecretStr | None = None confirmation_name: str = Field(min_length=1, max_length=63) class EnterRescueBody(BaseModel): model_config = ConfigDict(extra="forbid") template_id: str = Field(pattern=r"^[0-9a-f]{32}$") confirmation_name: str = Field(min_length=1, max_length=63) class ExitRescueBody(BaseModel): model_config = ConfigDict(extra="forbid") confirmation_name: str = Field(min_length=1, max_length=63) # PHP's billing ledger supplies the authoritative target. The original # XML was saved before rescue mode and may predate a traffic-quota block. network_suspended: bool = False class ForceRescueBody(BaseModel): model_config = ConfigDict(extra="forbid") confirmation_name: str = Field(min_length=1, max_length=63) ack_data_loss: Literal[True] class ActionBody(BaseModel): model_config = ConfigDict(extra="forbid") action: Literal["start", "shutdown", "reboot", "poweroff", "reset"] class CreateSnapshotBody(BaseModel): model_config = ConfigDict(extra="forbid") snapshot_key: str = Field(pattern=r"^[0-9a-f]{32}$") name: str = Field(min_length=1, max_length=64) consistency_mode: Literal["filesystem", "crash"] = "filesystem" @model_validator(mode="after") def validate_name(self): if (self.name != self.name.strip() or any(ord(char) < 32 or ord(char) == 127 for char in self.name)): raise ValueError("快照名称不能包含首尾空白或控制字符") return self class EmptySnapshotBody(BaseModel): model_config = ConfigDict(extra="forbid") class ServiceResumeBody(BaseModel): model_config = ConfigDict(extra="forbid") # Only restart a guest that the expiry policy itself stopped. A tenant # that had already shut the VM down before expiry must remain shut down # after renewal. start_if_suspended: bool = False class FirewallRuleBody(BaseModel): model_config = ConfigDict(extra="forbid") protocol: Literal["tcp", "udp", "icmp"] source_cidr: str = Field(min_length=1, max_length=18) port_from: int | None = Field(default=None, ge=1, le=65535) port_to: int | None = Field(default=None, ge=1, le=65535) description: str = Field(default="", max_length=120) @model_validator(mode="after") def validate_firewall_rule(self): try: source = ipaddress.ip_network(self.source_cidr, strict=False) except ValueError as exc: raise ValueError("来源必须是 IPv4 CIDR") from exc if source.version != 4: raise ValueError("防火墙一期仅支持 IPv4") if self.source_cidr != str(source): raise ValueError("来源必须使用规范 IPv4 CIDR") self.source_cidr = str(source) if self.protocol in {"tcp", "udp"}: if (self.port_from is None or self.port_to is None or self.port_from > self.port_to): raise ValueError("TCP/UDP 必须提供有效端口范围") elif self.port_from is not None or self.port_to is not None: raise ValueError("ICMP 规则不能包含端口") if (self.description != self.description.strip() or any(ord(char) < 32 or ord(char) == 127 for char in self.description)): raise ValueError("规则说明不能包含首尾空白或控制字符") return self class FirewallPolicyBody(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool revision: int = Field(ge=1, le=9223372036854775807) policy_digest: str = Field(pattern=r"^[0-9a-f]{64}$") rules: list[FirewallRuleBody] = Field(default_factory=list, max_length=50) def _public_host(profile: dict | None) -> dict: if profile is None: return {"configured": False} keys = ( "host", "port", "key_type", "fingerprint", "storage_pool", "iso_pool", "default_network", "private_key_configured", "private_key_encrypted", "key_locked", "private_key_passphrase_saved", ) return { "configured": True, **{key: profile[key] for key in keys if key in profile}, "rescue_iso_pool": profile.get("rescue_iso_pool", ""), "region": profile.get("region", ""), "public_pool_cidr": profile.get("public_pool_cidr", ""), "public_gateway": profile.get("public_gateway", ""), "public_dns_servers": profile.get("public_dns_servers", []), "public_reserved_ips": profile.get("public_reserved_ips", []), "public_vm_prefix": profile.get("public_vm_prefix"), } def _peer(request: Request) -> str: # Uvicorn has already processed forwarding headers from its trusted local # reverse proxy. Never read a raw client-supplied X-Real-IP here, because # that would let a login attacker reset the rate limit. return request.client.host if request.client else "unknown" def _origin_matches_host(origin: str | None, host: str | None) -> bool: if not origin or not host: return False try: parsed = urlsplit(origin) except ValueError: return False if parsed.scheme != "https": return False return bool( parsed.netloc == host and parsed.hostname and not parsed.username and not parsed.password and not parsed.path and not parsed.query and not parsed.fragment ) def _plain_loopback_allowed(scope: dict, host: str | None) -> bool: """Permit local diagnostics only when both peer and requested host are local. The request scheme is supplied by the ASGI server. Uvicorn's proxy header handling may upgrade it to HTTPS/WSS only for its explicitly trusted proxy address; this function never trusts an incoming forwarding header itself. """ client = scope.get("client") if not client or not host: return False try: peer_is_local = ipaddress.ip_address(client[0]).is_loopback host_is_local = urlsplit("//" + host).hostname in { "localhost", "127.0.0.1", "::1", } except ValueError: return False return peer_is_local and host_is_local def create_app( settings: Settings | None = None, *, panel_state: PanelState | None = None, remote_manager: RemoteHostManager | None = None, service_factory: Callable[[dict], VMService] | None = None, metric_store: Any | None = None, engine_components: EngineComponents | None = None, ) -> FastAPI: settings = settings or Settings.from_env() settings.validate() # Injected objects keep the existing isolated HTTP test harness working. # Normal production startup never uses the old JSON-backed stores. internal_mode = engine_components is not None database_mode = internal_mode or (panel_state is None and remote_manager is None) bootstrap = DatabaseBootstrap(settings.data_dir) if database_mode and not internal_mode else None data_store = None updater = None if internal_mode: state = engine_components.state manager = engine_components.manager templates = engine_components.templates data_store = engine_components.store metric_store = engine_components.metric_store setup_token = None elif database_mode: configured_database = bootstrap.load_config() if configured_database is None: setup_token = bootstrap.ensure_token() state = None manager = None templates = None else: state, manager, templates, data_store = make_components( settings.data_dir, configured_database, bootstrap.master_key(), settings.ssh_binary, ) setup_token = (bootstrap.ensure_token() if state.setup_required() else state.initialize_setup_token()) else: state = panel_state or PanelState(settings.data_dir) setup_token = state.initialize_setup_token() manager = remote_manager or RemoteHostManager( settings.data_dir, ssh_binary=settings.ssh_binary) templates = TemplateStore(settings.data_dir) if setup_token is not None: # Uvicorn writes this logger to stderr, which the supplied systemd # service captures in its journal. Never expose the token over HTTP. STARTUP_LOG.warning("VIRSH_PANEL_SETUP_TOKEN=%s", setup_token) if database_mode and (settings.data_dir / "generations").exists(): STARTUP_LOG.warning( "Legacy data/generations detected; its old SSH key files are not migrated " "or automatically removed. Verify and remove them separately." ) shared_cloud_template_runtime = ( engine_components.cloud_templates if internal_mode and engine_components.cloud_templates is not None else None ) import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="cloud-image-import") import_lock = threading.RLock() import_jobs: dict[str, dict] = {} pending_templates: set[tuple[str, str]] = set() rescue_catalog = RescueTemplateStore(data_store) rescue_import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="rescue-iso-import") rescue_import_lock = threading.RLock() rescue_import_jobs: dict[str, dict] = {} pending_rescue_imports: set[str] = set() shared_user_iso_runtime = ( engine_components.user_isos if internal_mode and engine_components.user_isos is not None else None ) if shared_user_iso_runtime is not None: user_iso_catalog = shared_user_iso_runtime.catalog user_iso_files = shared_user_iso_runtime.files user_iso_import_executor = shared_user_iso_runtime.executor user_iso_import_lock = shared_user_iso_runtime.lock user_iso_import_jobs = shared_user_iso_runtime.jobs pending_user_iso_imports = shared_user_iso_runtime.pending else: user_iso_catalog = UserISOStore(data_store) user_iso_files = UserISOFiles(settings.data_dir) user_iso_import_executor = ThreadPoolExecutor( max_workers=2, thread_name_prefix="user-iso-import") user_iso_import_lock = threading.RLock() user_iso_import_jobs: dict[str, dict] = {} pending_user_iso_imports: set[str] = set() # One child app represents one host. Serialize marker recheck, cache # transfer and marker commit so concurrent VM creates cannot both act on a # stale "not ready" observation for the same remote volume. user_iso_cache_lock = threading.RLock() cloud_template_cache_lock = threading.RLock() reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall") reinstall_lock = threading.RLock() reinstall_jobs: dict[str, dict] = {} pending_reinstalls: set[tuple[str, str]] = set() rescue_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-rescue") rescue_lock = threading.RLock() rescue_jobs: dict[str, dict] = {} pending_rescues: set[tuple[str, str]] = set() # A full local backup can consume all currently allocated guest blocks. # Serialize snapshot I/O per host so two sparse targets cannot both pass a # stale free-space observation and then fill the pool together. This is a # dedicated executor: unrelated VM reads and operations remain concurrent. snapshot_executor = ThreadPoolExecutor( max_workers=1, thread_name_prefix="vm-snapshot") snapshot_lock = threading.RLock() snapshot_jobs: dict[str, dict] = {} snapshot_futures: dict[str, Any] = {} pending_snapshot_jobs: set[tuple[str, str]] = set() snapshot_suspend_barriers: set[tuple[str, str]] = set() test_snapshots: dict[str, str] = {} test_rescue_states: dict[str, str] = {} test_firewalls: dict[str, str] = {} # The production store encrypts each credential record with a fresh AES-GCM # nonce. The in-memory fallback is only for injected, database-free tests. test_credentials: dict[str, str] = {} public_pool_catalog_lock = threading.RLock() public_pool_catalog_kind = "public_pool_catalog" public_pool_catalog_key = "current" def legacy_public_network_pool(profile: dict) -> dict | None: if not profile.get("public_pool_cidr"): return None try: return validate_public_network_pool({ "pool_key": "legacy-default", "revision": 1, "state": "active", "network_selector": profile["default_network"], "cidr": profile["public_pool_cidr"], "gateway": profile["public_gateway"], "dns_servers": profile.get("public_dns_servers", []), "reserved_ips": profile.get("public_reserved_ips", []), "guest_prefix_length": ( profile.get("public_vm_prefix") if profile.get("public_vm_prefix") is not None else ipaddress.IPv4Network( profile["public_pool_cidr"], strict=True).prefixlen ), }, profile["host"], allow_legacy_key=True) except (KeyError, TypeError, ValueError, RemoteHostError) as exc: raise VMError( "宿主机旧公网地址池配置损坏,请管理员重新保存设置", 500, ) from exc def read_native_public_network_pools( profile: dict, ) -> tuple[str | None, list[dict]]: if data_store is None: return None, [] raw = data_store.get(public_pool_catalog_kind, public_pool_catalog_key) if raw is None: return None, [] try: catalog = json.loads(raw) if (not isinstance(catalog, dict) or set(catalog) != {"version", "pools"} or catalog["version"] != 1 or not isinstance(catalog["pools"], list)): raise ValueError("invalid public pool catalog") pools = validate_public_network_pools(catalog["pools"], profile["host"]) except (KeyError, TypeError, ValueError, RemoteHostError) as exc: raise VMError( "宿主机公网地址池目录损坏,请管理员核查引擎记录", 500, ) from exc return raw, pools def public_network_pools(profile: dict) -> tuple[list[dict], bool]: raw, pools = read_native_public_network_pools(profile) if raw is not None: return pools, False legacy = legacy_public_network_pool(profile) return ([] if legacy is None else [legacy]), legacy is not None def public_pool_runtime_fields(pool: dict) -> dict: return { key: pool[key] for key in ( "network_selector", "cidr", "gateway", "dns_servers", "reserved_ips", "guest_prefix_length", ) } def public_pool_for_key( profile: dict, pool_key: str, *, active: bool = False, ) -> dict: pools, _compatibility = public_network_pools(profile) pool = next((item for item in pools if item["pool_key"] == pool_key), None) if pool is None: raise VMError("公网地址池不存在或尚未同步到宿主机", 409) if active and pool["state"] != "active": raise VMError("公网地址池没有开放新地址分配", 409) return pool def public_pool_for_ip( profile: dict, public_ip_cidr: str, *, active: bool = False, ) -> dict: try: requested = ipaddress.IPv4Interface(public_ip_cidr) except (TypeError, ValueError) as exc: raise VMError("预留公网 IP 格式无效", 422) from exc matches = [] pools, _compatibility = public_network_pools(profile) for pool in pools: network = ipaddress.IPv4Network(pool["cidr"], strict=True) if (requested.ip in network and requested.network.prefixlen == pool["guest_prefix_length"] and (not active or pool["state"] == "active")): matches.append(pool) if len(matches) != 1: raise VMError("预留公网 IP 无法唯一匹配宿主机地址池", 422) return matches[0] def ip_claim_key(profile: dict, address: str) -> str: return f"{host_scope(profile)}.{address}" def insert_public_ip_claim( profile: dict, address: str, vm_id: str, pool_key: str | None = None, ) -> bool: if data_store is None: return True claim_value = {"vm_uuid": vm_id, "reserved_at": int(time.time())} if pool_key is not None: claim_value["pool_key"] = pool_key claim = json.dumps(claim_value, separators=(",", ":")) return data_store.insert("ip_claim", ip_claim_key(profile, address), claim) def public_ip_claim_addresses(profile: dict) -> set[ipaddress.IPv4Address]: claimed: set[ipaddress.IPv4Address] = set() if data_store is None: return claimed scope_prefix = f"{host_scope(profile)}." for key in data_store.list("ip_claim"): if not key.startswith(scope_prefix): continue try: claimed.add(ipaddress.IPv4Address(key[len(scope_prefix):])) except ValueError: continue return claimed def public_network_pool_state( profile: dict, pool: dict, *, claimed_addresses: set[ipaddress.IPv4Address] | None = None, ) -> dict: """Build address state from the panel-owned claim ledger only. Dedicated hosts are configured out of band and the panel owns every sellable IP. Do not enumerate libvirt domains on this hot path; the atomic claim below is the engine-side concurrency guard. """ try: network = ipaddress.IPv4Network(pool["cidr"], strict=True) gateway = ipaddress.IPv4Address(pool["gateway"]) guest_prefix = pool["guest_prefix_length"] dns = [ipaddress.IPv4Address(value) for value in pool["dns_servers"]] reserved = [ipaddress.IPv4Address(value) for value in pool["reserved_ips"]] statically_blocked = { network.network_address, network.broadcast_address, gateway, *dns, *reserved, } try: host_address = ipaddress.IPv4Address(profile.get("host", "")) except (TypeError, ValueError): host_address = None if host_address is not None and host_address in network: statically_blocked.add(host_address) except (KeyError, TypeError, ValueError) as exc: raise VMError( "宿主机公网地址池配置损坏,请管理员修复设置", 500, ) from exc allocated: set[ipaddress.IPv4Address] = set() if claimed_addresses is None: claimed_addresses = public_ip_claim_addresses(profile) allocated.update(address for address in claimed_addresses if address in network) usable = set(network.hosts()) reserved_in_pool = usable & statically_blocked allocated_in_pool = (usable - reserved_in_pool) & allocated available = sorted(usable - reserved_in_pool - allocated_in_pool) return { **pool, "configuration_state": "ready", "pool_cidr": str(network), "guest_prefix": guest_prefix, "usable_count": len(usable), "reserved_count": len(reserved_in_pool), "allocated_count": len(allocated_in_pool), "available": available, "_network": network, "_usable": usable, "_reserved": reserved_in_pool, "_allocated": allocated_in_pool, } def allocate_public_ip(profile: dict, pool: dict, vm_id: str) -> str: """Choose and reserve a public IP, including across panel processes. Claims remain after a successful create and are released on panel VM deletion. A claim from an uncertain create is retained rather than risking reuse of an address whose guest may still exist. """ if pool["state"] != "active": raise VMError("公网地址池没有开放新地址分配", 409) state = public_network_pool_state(profile, pool) for address in state["available"]: if not insert_public_ip_claim( profile, str(address), vm_id, pool["pool_key"]): continue return f"{address}/{state['guest_prefix']}" raise VMError("公网地址池中没有可分配的 IP,请检查已用和保留地址", 409) def claim_scheduled_public_ip( profile: dict, pool: dict, public_ip_cidr: str, vm_id: str, ) -> str: """Atomically claim the exact pool address reserved by PHP. PHP owns the commercial reservation, while this engine claim closes the race between concurrent workers and libvirt domain creation. The host profile remains authoritative for prefix, gateway and DNS. """ if pool["state"] != "active": raise VMError("公网地址池没有开放新地址分配", 409) state = public_network_pool_state(profile, pool) try: requested = ipaddress.IPv4Interface(public_ip_cidr) except (TypeError, ValueError) as exc: raise VMError("预留公网 IP 格式无效", 422) from exc address = requested.ip if (requested.network.prefixlen != state["guest_prefix"] or address not in state["_network"] or address not in state["_usable"]): raise VMError("预留公网 IP 不属于宿主机可分配地址池", 422) if address in state["_reserved"]: raise VMError("预留公网 IP 是宿主机保留地址,不能分配", 409) if address in state["_allocated"]: raise VMError("预留公网 IP 已被虚拟机或其他任务占用", 409) if not insert_public_ip_claim( profile, str(address), vm_id, pool["pool_key"]): raise VMError("预留公网 IP 已被其他创建任务占用", 409) return f"{address}/{state['guest_prefix']}" def claim_explicit_public_ip(profile: dict, public_ip_cidr: str, vm_id: str) -> None: if data_store is None: return try: address = str(ipaddress.IPv4Interface(public_ip_cidr).ip) except (TypeError, ValueError): # The libvirt create path returns the detailed input error. return if not insert_public_ip_claim(profile, address, vm_id): raise VMError("该公网 IP 已被面板中的虚拟机或创建任务占用", 409) def release_public_ip_claim(profile: dict, public_ip_cidr: str, vm_id: str) -> None: if data_store is None: return try: address = str(ipaddress.IPv4Interface(public_ip_cidr).ip) key = ip_claim_key(profile, address) except (TypeError, ValueError, KeyError): return raw = data_store.get("ip_claim", key) if raw is None: return try: claim = json.loads(raw) except (TypeError, ValueError): LOG.error("Could not parse public IP claim for %s", key) return if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id: data_store.delete("ip_claim", key) def release_vm_public_ip_claims(profile: dict, vm_id: str) -> None: if data_store is None: return prefix = f"{host_scope(profile)}." for key, raw in data_store.list("ip_claim").items(): if not key.startswith(prefix): continue try: claim = json.loads(raw) except (TypeError, ValueError): LOG.error("Could not parse public IP claim for %s", key) continue if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id: data_store.delete("ip_claim", key) def compare_and_set_public_pool_catalog( expected: str | None, value: str, ) -> bool: if data_store is None: raise VMError("公网地址池目录存储不可用", 409) compare = getattr(data_store, "compare_and_set", None) if callable(compare): return bool(compare( public_pool_catalog_kind, public_pool_catalog_key, expected, value, )) # Database-free tests and the retired single-process browser mode use # a simple store. Keep the same compare-and-set behavior locally. with public_pool_catalog_lock: current = data_store.get( public_pool_catalog_kind, public_pool_catalog_key) if current != expected: return False data_store.set(public_pool_catalog_kind, public_pool_catalog_key, value) return True def upsert_public_network_pool( profile: dict, pool_key: str, body: PublicNetworkPoolBody, ) -> tuple[dict, bool]: if not PUBLIC_POOL_KEY.fullmatch(pool_key): raise VMError("公网地址池标识必须是 32 位小写十六进制", 422) try: requested = validate_public_network_pool({ "pool_key": pool_key, **body.model_dump(), }, profile["host"]) except RemoteHostError as exc: raise VMError(str(exc), exc.status) from exc # The data model stores a selector now so a later multi-line release # does not need another migration. Network-sensitive VM operations still # instantiate the host's default VMService, so accepting another bridge # today would create guests that cannot be safely reinstalled or edited. if requested["network_selector"] != profile["default_network"]: raise VMError( "当前阶段公网地址池必须使用宿主机默认网络;多线路网桥尚未启用", 409, ) for _attempt in range(8): raw, native = read_native_public_network_pools(profile) existing = next( (item for item in native if item["pool_key"] == pool_key), None) if existing is not None: if requested["revision"] < existing["revision"]: raise VMError("公网地址池版本已更新,请刷新后重试", 409) if requested["revision"] == existing["revision"]: if requested == existing: return existing, True raise VMError("相同公网地址池版本对应了不同配置", 409) candidate = [item for item in native if item["pool_key"] != pool_key] candidate.append(requested) try: candidate = validate_public_network_pools(candidate, profile["host"]) except RemoteHostError as exc: raise VMError(str(exc), exc.status) from exc claimed_addresses = public_ip_claim_addresses(profile) if existing is not None and ( public_pool_runtime_fields(existing) != public_pool_runtime_fields(requested)): state = public_network_pool_state( profile, existing, claimed_addresses=claimed_addresses) if state["allocated_count"]: raise VMError( "公网地址池仍有虚拟机或任务占用,不能修改网段、网关、DNS或保留地址", 409, ) if raw is None: legacy = legacy_public_network_pool(profile) if (legacy is not None and public_pool_runtime_fields(legacy) != public_pool_runtime_fields(requested)): state = public_network_pool_state( profile, legacy, claimed_addresses=claimed_addresses) if state["allocated_count"]: raise VMError( "旧公网地址池仍有虚拟机或任务占用;首个新地址池必须沿用原网络配置", 409, ) encoded = json.dumps( {"version": 1, "pools": candidate}, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) if compare_and_set_public_pool_catalog(raw, encoded): return requested, False raise VMError("公网地址池配置正在被其他请求修改,请刷新后重试", 409) def credential_key(profile: dict, vm_id: str) -> str: try: canonical_id = str(uuid.UUID(vm_id)) return f"{host_scope(profile)}.{canonical_id}" except (TypeError, ValueError, AttributeError, KeyError) as exc: raise VMError("虚拟机或宿主机身份信息无效", 400) from exc def save_credential(profile: dict, vm_id: str, auth_mode: str, generation: str, password: str | None = None) -> None: key = credential_key(profile, vm_id) if not isinstance(generation, str) or not generation: raise ValueError("虚拟机系统配置代号无效") value = json.dumps({"auth_mode": auth_mode, "password": password, "generation": generation}, ensure_ascii=False, separators=(",", ":")) if data_store is not None: data_store.set("vm_credential", key, value) else: test_credentials[key] = value def delete_credential(profile: dict, vm_id: str) -> None: key = credential_key(profile, vm_id) if data_store is not None: data_store.delete("vm_credential", key) else: test_credentials.pop(key, None) def rescue_state_key(profile: dict, vm_id: str) -> str: return credential_key(profile, vm_id) def rescue_xml_matches(first: str, second: str) -> bool: try: return ET.canonicalize(first) == ET.canonicalize(second) except (TypeError, ET.ParseError): return False def load_rescue_state(profile: dict, vm_id: str) -> dict | None: key = rescue_state_key(profile, vm_id) raw = (data_store.get("vm_rescue", key) if data_store is not None else test_rescue_states.get(key)) if raw is None: return None try: record = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("虚拟机救援记录损坏") from exc if (not isinstance(record, dict) or not isinstance(record.get("original_xml"), str) or not record["original_xml"].startswith(" None: key = rescue_state_key(profile, vm_id) value = json.dumps(record, ensure_ascii=False, separators=(",", ":")) if data_store is not None: data_store.set("vm_rescue", key, value) else: test_rescue_states[key] = value def delete_rescue_state(profile: dict, vm_id: str) -> None: key = rescue_state_key(profile, vm_id) if data_store is not None: data_store.delete("vm_rescue", key) else: test_rescue_states.pop(key, None) def current_rescue_state(profile: dict, vm_id: str, service: VMService | None = None) -> dict | None: """Clear a stale record only when the original inactive XML is intact. An interrupted worker may have stopped before changing the VM. The database job is marked failed on restart, but its recovery record must not indefinitely block normal operations in that safe case. Callers hold rescue_lock so this cannot race a live rescue worker. """ record = load_rescue_state(profile, vm_id) if record is None or (host_scope(profile), vm_id) in pending_rescues: return record service = service or new_service(profile) current_vm = service.get_vm(vm_id) if current_vm.get("rescue_mode") or current_vm.get("rescue_staged"): return record try: original_still_defined = rescue_xml_matches( service.prepare_rescue_vm(vm_id, record["volume_name"]), record["original_xml"], ) except VMError: return record if original_still_defined: delete_rescue_state(profile, vm_id) return None return record def parse_credential(raw: str | None) -> dict | None: if raw is None: return None try: value = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("虚拟机登录凭据记录损坏") from exc if (not isinstance(value, dict) or value.get("auth_mode") not in ("password", "ssh_key", "ssh_key_password") or (value["auth_mode"] in ("password", "ssh_key_password") and (not isinstance(value.get("password"), str) or not value["password"])) or not isinstance(value.get("generation"), str) or not value["generation"] or (value["auth_mode"] == "ssh_key" and value.get("password") is not None)): raise DatabaseError("虚拟机登录凭据记录损坏") return value def get_credential(profile: dict, vm_id: str) -> dict | None: key = credential_key(profile, vm_id) raw = (data_store.get("vm_credential", key) if data_store is not None else test_credentials.get(key)) return parse_credential(raw) def firewall_record_key(profile: dict, vm_id: str) -> str: return credential_key(profile, vm_id) def load_firewall_record(profile: dict, vm_id: str) -> dict | None: key = firewall_record_key(profile, vm_id) raw = (data_store.get("vm_firewall", key) if data_store is not None else test_firewalls.get(key)) if raw is None: return None try: record = json.loads(raw) policy = FirewallPolicyBody.model_validate({ "enabled": record["enabled"], "revision": record["revision"], "policy_digest": record["policy_digest"], "rules": record["rules"], }) except (KeyError, TypeError, ValueError) as exc: raise DatabaseError("虚拟机防火墙记录损坏") from exc record_version = record.get("version") if isinstance(record, dict) else None if (not isinstance(record, dict) or record_version not in {1, 2} or record.get("vm_id") != str(uuid.UUID(vm_id)) or not re.fullmatch(r"[0-9a-f]{64}", record.get("policy_digest", "")) or not re.fullmatch(r"[0-9a-f]{64}", record.get("filter_digest", "")) or isinstance(record.get("applied_at"), bool) or not isinstance(record.get("applied_at"), int)): raise DatabaseError("虚拟机防火墙记录损坏") filter_version = record.get("filter_version", 1) if (isinstance(filter_version, bool) or not isinstance(filter_version, int) or filter_version not in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS): raise DatabaseError("虚拟机防火墙记录损坏") try: platform_policy = ( normalize_platform_policy_snapshot( record["platform_policy_snapshot"]) if record_version == 2 else default_platform_policy_snapshot() ) except (KeyError, TypeError, ValueError) as exc: raise DatabaseError("虚拟机防火墙记录损坏") from exc if (record_version == 2 and (not isinstance(record.get("platform_policy_digest"), str) or not hmac.compare_digest( record["platform_policy_digest"], platform_policy["digest"], ))): raise DatabaseError("虚拟机防火墙记录损坏") if record_version == 1 and filter_version >= 4: raise DatabaseError("虚拟机防火墙记录损坏") normalized_policy = policy.model_dump() if not hmac.compare_digest( normalized_policy["policy_digest"], firewall_policy_digest( normalized_policy["enabled"], normalized_policy["rules"], )): raise DatabaseError("虚拟机防火墙记录损坏") expected_filter_digest = build_firewall_filter_xml( vm_id, normalized_policy["enabled"], normalized_policy["rules"], compiler_version=filter_version, platform_policy_snapshot=platform_policy, )[3] if not hmac.compare_digest( record["filter_digest"], expected_filter_digest): raise DatabaseError("虚拟机防火墙记录损坏") return { **record, **normalized_policy, "version": record_version, "filter_version": filter_version, "platform_policy_snapshot": platform_policy, "platform_policy_digest": platform_policy["digest"], } def save_firewall_record(profile: dict, vm_id: str, record: dict) -> None: key = firewall_record_key(profile, vm_id) encoded = json.dumps( record, ensure_ascii=False, separators=(",", ":")) if data_store is not None: data_store.set("vm_firewall", key, encoded) else: test_firewalls[key] = encoded def delete_firewall_record(profile: dict, vm_id: str) -> None: key = firewall_record_key(profile, vm_id) if data_store is not None: data_store.delete("vm_firewall", key) else: test_firewalls.pop(key, None) def is_default_platform_firewall( vm_id: str, observed: dict, ) -> bool: """Recognize the exact record-free wrapper installed at VM creation.""" observed_digest = observed.get("filter_digest") platform_policy = observed.get("platform_policy_snapshot") expected_digest = build_firewall_filter_xml( vm_id, False, [], platform_policy_snapshot=platform_policy)[3] return bool( observed.get("filter_mode") == "managed" and observed.get("sync_state") == "active" and observed.get("coherent") is True and isinstance(observed_digest, str) and hmac.compare_digest(observed_digest, expected_digest) ) def upgrade_firewall_filter_if_needed( profile: dict, vm_id: str, service: VMService, ) -> bool: """Verify the host filter and recompile trusted legacy policy only.""" record = load_firewall_record(profile, vm_id) observed = service.get_firewall_policy(vm_id) observed_filter_digest = observed.get("filter_digest") observed_mode = observed.get("filter_mode") try: observed_platform_policy = normalize_platform_policy_snapshot( observed.get("platform_policy_snapshot") or default_platform_policy_snapshot()) except ValueError as exc: raise VMError( "虚拟机平台网络策略证据无效", 409, error_code="firewall_drift_detected", ) from exc observed_active = ( observed.get("sync_state") == "active" and observed.get("coherent") is True) if record is None: if (observed_active and observed_mode == "baseline" and observed_filter_digest is None): return False if is_default_platform_firewall(vm_id, observed): return False raise VMError( "防火墙实际状态与已保存版本不一致,请先核查漂移", 409, error_code="firewall_drift_detected", ) observation_active = ( observed_active and observed_mode == "managed" and isinstance(observed_filter_digest, str)) if not hmac.compare_digest( record["platform_policy_digest"], observed_platform_policy["digest"]): raise VMError( "虚拟机平台网络策略与引擎记录不一致", 409, error_code="firewall_drift_detected", ) if record["filter_version"] >= FIREWALL_FILTER_COMPILER_VERSION: if (observation_active and hmac.compare_digest( observed_filter_digest, record["filter_digest"])): return False raise VMError( "防火墙实际状态与已保存版本不一致,请先核查漂移", 409, error_code="firewall_drift_detected", ) expected_filter_digest = build_firewall_filter_xml( vm_id, record["enabled"], record["rules"], platform_policy_snapshot=observed_platform_policy)[3] if not observation_active: raise VMError( "防火墙实际状态与已保存版本不一致,请先核查漂移", 409, error_code="firewall_drift_detected", ) if hmac.compare_digest( observed_filter_digest, expected_filter_digest): upgraded = { **record, "version": 2, "filter_version": FIREWALL_FILTER_COMPILER_VERSION, "filter_digest": expected_filter_digest, "platform_policy_snapshot": observed_platform_policy, "platform_policy_digest": observed_platform_policy["digest"], "applied_at": int(time.time()), } try: save_firewall_record(profile, vm_id, upgraded) except Exception as exc: raise VMError( "防火墙已升级但本地证据未确认,已停止修改网络", 503, error_code="firewall_result_uncertain", ) from exc return True if not hmac.compare_digest( observed_filter_digest, record["filter_digest"]): raise VMError( "防火墙实际状态与已保存版本不一致,请先核查漂移", 409, error_code="firewall_drift_detected", ) result = service.apply_firewall_policy( vm_id, enabled=record["enabled"], rules=record["rules"]) result_policy_digest = result.get("policy_digest") result_filter_digest = result.get("filter_digest") result_platform_digest = result.get("platform_policy_digest") if (not isinstance(result_policy_digest, str) or not isinstance(result_filter_digest, str) or not hmac.compare_digest( result_policy_digest, record["policy_digest"]) or not hmac.compare_digest( result_filter_digest, expected_filter_digest) or not isinstance(result_platform_digest, str) or not hmac.compare_digest( result_platform_digest, observed_platform_policy["digest"])): raise VMError( "防火墙编译器升级结果不确定,已停止修改网络", 503, error_code="firewall_result_uncertain", ) upgraded = { **record, "version": 2, "filter_version": FIREWALL_FILTER_COMPILER_VERSION, "filter_digest": expected_filter_digest, "platform_policy_snapshot": observed_platform_policy, "platform_policy_digest": observed_platform_policy["digest"], "applied_at": int(time.time()), } try: save_firewall_record(profile, vm_id, upgraded) except Exception as exc: raise VMError( "防火墙已升级但本地证据未确认,已停止修改网络", 503, error_code="firewall_result_uncertain", ) from exc return True def snapshot_record_key(vm_id: str, snapshot_key: str) -> str: try: canonical_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc if not isinstance(snapshot_key, str) or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key): raise VMError("快照标识无效", 422) return f"{canonical_id}:{snapshot_key}" def parse_snapshot_record(key: str, raw: str) -> dict: try: record = json.loads(raw) vm_id, snapshot_key = key.split(":", 1) canonical_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise DatabaseError("虚拟机快照记录损坏") from exc allowed_states = { "creating", "ready", "restoring", "deleting", "failed", "uncertain", } if (not isinstance(record, dict) or record.get("vm_id") != canonical_id or record.get("snapshot_key") != snapshot_key or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key) or record.get("volume_name") != snapshot_volume_name( canonical_id, snapshot_key) or record.get("state") not in allowed_states or record.get("consistency") not in {"filesystem", "crash"} or not isinstance(record.get("name"), str) or not 1 <= len(record["name"]) <= 64 or not isinstance(record.get("created_at"), int) or record["created_at"] < 1): raise DatabaseError("虚拟机快照记录损坏") size_bytes = record.get("size_bytes") if (size_bytes is not None and (isinstance(size_bytes, bool) or not isinstance(size_bytes, int) or size_bytes < 0)): raise DatabaseError("虚拟机快照记录损坏") return record def list_snapshot_records(vm_id: str) -> list[dict]: canonical_id = str(uuid.UUID(vm_id)) prefix = canonical_id + ":" rows = (data_store.list("vm_snapshot") if data_store is not None else test_snapshots.copy()) records = [ parse_snapshot_record(key, raw) for key, raw in rows.items() if key.startswith(prefix) ] return sorted(records, key=lambda row: (row["created_at"], row["snapshot_key"]), reverse=True) def has_retained_snapshots() -> bool: rows = (data_store.list("vm_snapshot") if data_store is not None else test_snapshots.copy()) return any( parse_snapshot_record(key, raw)["state"] in { "ready", "creating", "restoring", "deleting", "uncertain", } for key, raw in rows.items() ) def get_snapshot_record(vm_id: str, snapshot_key: str) -> dict: key = snapshot_record_key(vm_id, snapshot_key) raw = (data_store.get("vm_snapshot", key) if data_store is not None else test_snapshots.get(key)) if raw is None: raise VMError("快照不存在", 404) return parse_snapshot_record(key, raw) def save_snapshot_record(record: dict) -> None: key = snapshot_record_key(record["vm_id"], record["snapshot_key"]) value = json.dumps(record, ensure_ascii=False, separators=(",", ":")) if data_store is not None: data_store.set("vm_snapshot", key, value) else: test_snapshots[key] = value def insert_snapshot_record(record: dict) -> bool: key = snapshot_record_key(record["vm_id"], record["snapshot_key"]) value = json.dumps(record, ensure_ascii=False, separators=(",", ":")) if data_store is not None: return bool(data_store.insert("vm_snapshot", key, value)) if key in test_snapshots: return False test_snapshots[key] = value return True def delete_snapshot_record(vm_id: str, snapshot_key: str) -> None: key = snapshot_record_key(vm_id, snapshot_key) if data_store is not None: data_store.delete("vm_snapshot", key) else: test_snapshots.pop(key, None) def public_snapshot_record(record: dict) -> dict: return { key: record[key] for key in ( "snapshot_key", "name", "state", "consistency", "created_at", "completed_at", "size_bytes", "last_restored_at", ) if key in record } def forbid_pending_snapshot( profile: dict, vm_id: str, label: str, *, allow_uncertain: bool = False, ignore_suspend_barrier: bool = False, ) -> None: key = (host_scope(profile), str(uuid.UUID(vm_id))) with snapshot_lock: if key in snapshot_suspend_barriers and not ignore_suspend_barrier: raise VMError( f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409) if key in pending_snapshot_jobs: raise VMError(f"虚拟机正在执行快照任务,暂时不能{label}", 409) blocked_states = {"creating", "restoring", "deleting"} if not allow_uncertain: blocked_states.add("uncertain") if any(row["state"] in blocked_states for row in list_snapshot_records(key[1])): raise VMError( "虚拟机存在进行中或结果不确定的快照任务,必须先完成安全核对", 409, ) def forbid_snapshot_suspend_barrier( profile: dict, vm_id: str, label: str, ) -> None: key = (host_scope(profile), str(uuid.UUID(vm_id))) with snapshot_lock: if key in snapshot_suspend_barriers: raise VMError( f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409) def forbid_retained_snapshot(profile: dict, vm_id: str, label: str) -> None: """Prevent replacing vda while a local recovery point is retained. Instance settings may safely rotate the current cloud-init seed: a later restore intentionally applies that current seed to the restored disk. Reinstall is different because it replaces the disk baseline that every retained local snapshot belongs to. """ canonical_id = str(uuid.UUID(vm_id)) forbid_pending_snapshot(profile, canonical_id, label) with snapshot_lock: if any(row["state"] != "failed" for row in list_snapshot_records(canonical_id)): raise VMError( f"虚拟机仍保留本地快照,请删除全部快照后再{label}", 409, ) def rollback_cancelled_snapshot_catalog(job: dict) -> None: """Roll back a snapshot row after its Future was cancelled queued. The durable ``cancelled_before_start`` marker proves that the executor accepted ``Future.cancel()`` before this helper runs. Consequently no libvirt mutation can have started, and the admission-state row can be restored without inspecting or changing the host. """ vm_id = job.get("vm_id") snapshot_key = job.get("snapshot_key") action = job.get("action") if (not isinstance(vm_id, str) or not isinstance(snapshot_key, str) or action not in {"create", "restore", "delete"}): raise DatabaseError("取消的快照任务身份无效") try: record = get_snapshot_record(vm_id, snapshot_key) except VMError as exc: if action == "create" and exc.status == 404: return raise DatabaseError("取消的快照登记记录不存在") from exc expected_state = { "create": "creating", "restore": "restoring", "delete": "deleting", }[action] if action == "create": if record["state"] != expected_state: raise DatabaseError("取消的快照创建登记状态已变化") delete_snapshot_record(vm_id, snapshot_key) try: get_snapshot_record(vm_id, snapshot_key) except VMError as exc: if exc.status == 404: return raise raise DatabaseError("取消的快照创建登记未能删除") if record["state"] not in {expected_state, "ready"}: raise DatabaseError("取消的快照登记状态已变化") if record["state"] != "ready": record["state"] = "ready" save_snapshot_record(record) if get_snapshot_record(vm_id, snapshot_key)["state"] != "ready": raise DatabaseError("取消的快照登记未能恢复") def finish_cancelled_snapshot_job(job_id: str, job: dict) -> dict: """Finish the second phase of a durable pre-start cancellation.""" if (job.get("cancelled_before_start") is not True or job.get("cancel_reason") != "network_suspend"): raise DatabaseError("快照取消凭据无效") rollback_cancelled_snapshot_catalog(job) terminal = dict(job) terminal.update( status="failed", phase="排队快照已由到期网络暂停策略取消", error="快照任务在开始前已取消,未调用虚拟化操作", cancelled_at=int(time.time()), finished_at=int(time.time()), ) terminal.pop("result", None) terminal.pop("error_code", None) # The terminal job is persisted only after catalog rollback. If this # write fails, the durable phase-one tombstone remains queued and both # startup recovery and operation polling can retry this exact step. save_job("snapshot_job", job_id, terminal) snapshot_jobs[job_id] = terminal pending_snapshot_jobs.discard((terminal["scope"], terminal["vm_id"])) snapshot_futures.pop(job_id, None) return dict(terminal) def cancel_queued_snapshot_for_network_suspend( profile: dict, vm_id: str, ) -> None: """Cancel exactly one queued same-VM job without touching libvirt.""" scope = host_scope(profile) matches = [ (job_id, job) for job_id, job in snapshot_jobs.items() if (job.get("scope") == scope and job.get("vm_id") == vm_id and job.get("status") in {"queued", "running"}) ] if len(matches) > 1: raise VMError("虚拟机存在多个快照任务,拒绝自动取消", 409) if matches: job_id, job = matches[0] if job.get("status") != "queued": raise VMError("虚拟机快照任务已经开始,暂时不能暂停网络", 409) if job.get("cancelled_before_start") is not True: future = snapshot_futures.get(job_id) if future is None or not future.cancel(): # A Future becomes RUNNING before its callable acquires # snapshot_lock. Never infer safety from the still-queued # job row when the executor has already claimed it. raise VMError( "虚拟机快照任务已经由执行器领取,暂时不能暂停网络", 409) tombstone = dict(job) tombstone.update( cancelled_before_start=True, cancel_reason="network_suspend", cancel_requested_at=int(time.time()), phase="正在取消尚未开始的排队快照", ) # Future.cancel() is already definitive process-local proof. # Publish the tombstone in memory first so a transient phase- # one database failure remains retryable by operation polling # without ever letting the cancelled callable reach libvirt. snapshot_jobs[job_id] = tombstone job = tombstone # Persist proof of Future.cancel() before changing the catalog. try: save_job("snapshot_job", job_id, tombstone) except Exception as exc: LOG.exception( "Could not persist queued snapshot cancellation: job=%s uuid=%s", job_id, vm_id, ) raise VMError( "排队快照已停止,但取消凭据尚未持久化,暂时不能暂停网络", 503, ) from exc try: finish_cancelled_snapshot_job(job_id, job) except Exception as exc: LOG.exception( "Could not finish queued snapshot cancellation: job=%s uuid=%s", job_id, vm_id, ) raise VMError( "排队快照已停止,但登记回滚尚未完成,暂时不能暂停网络", 503, ) from exc if (scope, vm_id) in pending_snapshot_jobs: raise VMError("虚拟机快照任务状态尚未收敛,暂时不能暂停网络", 409) def decorate_vm(vm: dict, credential: dict | None, profile: dict) -> dict: cloud = bool(vm.get("managed") and vm.get("os_source") == "cloud_template") current_credential = (credential if cloud and credential and credential["generation"] == vm.get("credential_generation") else None) return { **{key: value for key, value in vm.items() if key != "credential_generation"}, "region": profile.get("region", ""), "login_name": "root" if cloud else None, "auth_mode": current_credential["auth_mode"] if current_credential else "unknown", "password_available": bool(current_credential and current_credential["auth_mode"] in ( "password", "ssh_key_password")), } def save_job(kind: str, job_id: str, job: dict) -> None: delegated = TRUSTED_ACTOR.get() if internal_mode else None if delegated and delegated.get("operation_id"): job.setdefault("operation_id", delegated["operation_id"]) if data_store is not None: data_store.set(kind, job_id, json.dumps( job, ensure_ascii=False, separators=(",", ":"))) def delete_job(kind: str, job_id: str) -> None: if data_store is not None: data_store.delete(kind, job_id) def save_user_iso_job(job_id: str, job: dict) -> None: if shared_user_iso_runtime is not None: shared_user_iso_runtime.save_job(job_id, job) else: save_job("user_iso_import_job", job_id, job) def delete_user_iso_job(job_id: str) -> None: if shared_user_iso_runtime is not None: shared_user_iso_runtime.delete_job(job_id) else: delete_job("user_iso_import_job", job_id) def restore_jobs() -> None: """Restore interrupted jobs without pretending mutations resumed.""" if data_store is None: return pending_reinstalls.clear() destinations = [ ("reinstall_job", reinstall_jobs), ("rescue_import_job", rescue_import_jobs), ("rescue_job", rescue_jobs), ("snapshot_job", snapshot_jobs), ] if shared_cloud_template_runtime is None: destinations.append(("import_job", import_jobs)) if shared_user_iso_runtime is None: destinations.append(("user_iso_import_job", user_iso_import_jobs)) for kind, destination in destinations: for job_id, raw in data_store.list(kind).items(): try: job = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("数据库任务记录损坏") from exc if not isinstance(job, dict): raise DatabaseError("数据库任务记录损坏") if (kind == "reinstall_job" and job.get("status") in { "queued", "running", "succeeded", "completed", "done", "failed", "cancelled", "canceled", "uncertain", } and reinstall_post_commit(job)): # Releases before the cleanup boundary could persist a # warning/phase without the explicit marker. Upgrade any # such durable shape on startup so a direct engine poll # cannot present it as a completed or retryable reinstall. cleanup_pending = reinstall_cleanup_pending(job) job["status"] = "uncertain" job["phase"] = ( "重装完成,但旧系统盘清理待核对" if cleanup_pending else "重装结果待核对" ) job["error_code"] = ( REINSTALL_CLEANUP_PENDING_ERROR_CODE if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE ) job["error"] = ( "重装已完成,但旧系统盘清理待核对" if cleanup_pending else "重装结果待核对" ) job["reinstall_committed"] = True if cleanup_pending: job["cleanup_pending"] = True job.pop("finished_at", None) save_job(kind, job_id, job) if (kind == "snapshot_job" and job.get("status") == "queued" and job.get("cancelled_before_start") is True): # Phase one proves the previous process cancelled the # Future before its callable began. Finish only the local # catalog rollback; never reinterpret this as an # interrupted libvirt mutation. try: job = finish_cancelled_snapshot_job(job_id, job) except Exception: LOG.exception( "Could not finish durable queued snapshot cancellation: %s", job_id, ) destination[job_id] = job continue if (kind == "snapshot_job" and job.get("action") == "create" and job.get("consistency") == "filesystem" and job.get("guest_freeze_state") == "attempted"): try: profile = manager.current() if profile is None or profile.get("key_locked"): raise VMError( "宿主机未配置或密钥已锁定,无法恢复访客文件系统") recovery_service = ( service_factory(profile) if service_factory is not None else VMService( profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"], profile["default_network"], rescue_iso_pool=profile.get( "rescue_iso_pool", ""), ) ) recovery_service.thaw_guest_filesystems(job["vm_id"]) except Exception: job["freeze_recovery"] = "failed" LOG.exception( "Could not recover interrupted guest freeze: " "job=%s uuid=%s", job_id, job.get("vm_id"), ) else: job["guest_freeze_state"] = "recovered" job["freeze_recovery"] = "confirmed" job["freeze_recovered_at"] = int(time.time()) save_job(kind, job_id, job) if (kind == "reinstall_job" and job.get("status") == "running"): # The replacement domain was durably observed as started, # or may have crossed that boundary before the process # ended. Preserve an uncertain result in either case; # a durable post-commit phase or marker authorizes PHP # to copy the replacement credential/spec into its cache. committed = reinstall_post_commit(job) cleanup_pending = reinstall_cleanup_pending(job) job["status"] = "uncertain" job["phase"] = ( "重装完成,但旧系统盘清理待核对" if cleanup_pending else "重装结果待核对" ) # A stale terminal timestamp must not make the recovered # operation look settled while its physical result is # still awaiting inspection. job.pop("finished_at", None) if committed: job["error_code"] = ( REINSTALL_CLEANUP_PENDING_ERROR_CODE if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE ) job["error"] = ( "重装已完成,但旧系统盘清理待核对" if cleanup_pending else "重装结果待核对;面板重启后未能确认旧存储卷清理状态" ) job["reinstall_committed"] = True if cleanup_pending: job["cleanup_pending"] = True else: for key in ("error_code", "cleanup_pending", "warning", "generated_password", "reinstall_committed"): job.pop(key, None) job["error"] = "重装结果待核对;面板重启后无法确认新系统是否已启动" save_job(kind, job_id, job) elif job.get("status") in {"queued", "running"}: if kind == "snapshot_job": # A push backup or disk switch can continue or commit # after the HTTP process disappears. Never convert that # ambiguity into a retryable terminal failure. job["status"] = "uncertain" job["phase"] = "引擎重启,快照结果待核查" job["error"] = "引擎在快照任务期间重启;必须核查存储卷和虚拟机定义,不能重复提交" try: record = get_snapshot_record( job["vm_id"], job["snapshot_key"]) record["state"] = "uncertain" save_snapshot_record(record) except (KeyError, VMError, DatabaseError, ValueError): LOG.exception( "Could not mark interrupted snapshot record uncertain: %s", job_id) else: job["status"] = "failed" job["phase"] = "面板重启,任务已中断" job["error"] = "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交" for key in ("generated_password", "reinstall_committed", "cleanup_pending", "error_code", "warning"): job.pop(key, None) save_job(kind, job_id, job) elif kind == "snapshot_job" and job.get("status") == "uncertain": # A previous process may have persisted the job outcome # before it could mirror that state into the catalog. # The durable operation identity exists, so preserve the # row for evidence-only reconciliation rather than # treating it as the pre-queue crash window below. try: record = get_snapshot_record( job["vm_id"], job["snapshot_key"]) if record["state"] in { "creating", "restoring", "deleting"}: record["state"] = "uncertain" save_snapshot_record(record) except (KeyError, VMError, DatabaseError, ValueError): LOG.exception( "Could not align uncertain snapshot catalog row: %s", job_id, ) if (kind == "reinstall_job" and job.get("status") in {"queued", "running", "uncertain"} and isinstance(job.get("scope"), str) and isinstance(job.get("vm_id"), str)): # Rebuild the in-process conflict barrier after a panel # restart. Other VM mutation endpoints consult this set # before touching the same guest. pending_reinstalls.add((job["scope"], job["vm_id"])) destination[job_id] = job # The registry row is reserved before the durable job row. A process # loss in that narrow window otherwise leaves a permanent # creating/restoring/deleting record with no operation to reconcile. # No libvirt mutation is invoked until queue_snapshot_job has durably # saved its job row. Therefore a transitional catalog row with no # corresponding durable job proves the process died before the worker # could cross a physical mutation boundary. durable_snapshot_records = { (job.get("vm_id"), job.get("snapshot_key")) for job in snapshot_jobs.values() } for key, raw in data_store.list("vm_snapshot").items(): record = parse_snapshot_record(key, raw) identity = (record["vm_id"], record["snapshot_key"]) if (record["state"] in {"creating", "restoring", "deleting"} and identity not in durable_snapshot_records): if record["state"] == "creating": delete_snapshot_record(*identity) else: record["state"] = "ready" record["interrupted_at"] = int(time.time()) save_snapshot_record(record) LOG.warning( "Rolled back snapshot registry row created before durable job: " "uuid=%s snapshot=%s", *identity, ) restore_jobs() sessions = DatabaseSessionStore(data_store) if data_store is not None else SessionStore() login_limiter = LoginLimiter() registration_limiter = LoginLimiter() code_send_limiter = LoginLimiter() setup_limiter = LoginLimiter() unlock_limiter = LoginLimiter() def metric_profile(): if manager is None: return None background = getattr(manager, "current_for_background", None) return background() if callable(background) else manager.current() metric_collector = MetricsCollector( lambda: metric_store if metric_store is not None else data_store, metric_profile, lambda profile: new_service(profile), owner_getter=lambda scope, vm_id: state.users.owner_of_vm(scope, vm_id), ) if database_mode else None @asynccontextmanager async def lifespan(_app: FastAPI): if metric_collector is not None: metric_collector.start() try: yield finally: if metric_collector is not None: metric_collector.close() reinstall_executor.shutdown(wait=True, cancel_futures=False) snapshot_executor.shutdown(wait=True, cancel_futures=False) import_executor.shutdown(wait=True, cancel_futures=False) rescue_executor.shutdown(wait=True, cancel_futures=False) rescue_import_executor.shutdown(wait=True, cancel_futures=False) if shared_user_iso_runtime is None: user_iso_import_executor.shutdown(wait=True, cancel_futures=False) close_manager = getattr(manager, "close", None) if manager is not None else None if callable(close_manager): close_manager() app = FastAPI(title="Virsh Panel API", docs_url=None, redoc_url=None, lifespan=lifespan) app.mount("/static", StaticFiles(directory=str(settings.static_dir)), name="static") app.mount( "/vendor/novnc", StaticFiles(directory=str(settings.static_dir / "vendor" / "novnc")), name="novnc", ) @app.middleware("http") async def security_headers(request: Request, call_next): if not internal_mode and request.url.scheme != "https" and not ( request.url.path == "/api/v1/health" and request.url.scheme == "http" and _plain_loopback_allowed(request.scope, request.headers.get("host")) ): return JSONResponse(status_code=400, content={"error": "请使用 HTTPS 访问面板"}) response = await call_next(request) response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Referrer-Policy"] = "no-referrer" response.headers["X-Frame-Options"] = "DENY" response.headers["Content-Security-Policy"] = ( "default-src 'self'; script-src 'self'; style-src 'self'; " "img-src 'self' data:; connect-src 'self' wss:; " "object-src 'none'; frame-ancestors 'none'; base-uri 'self'" ) if (request.url.path == "/" or request.url.path.startswith("/api/") or request.url.path in ("/static/app.js", "/static/styles.css")): response.headers["Cache-Control"] = "no-store" return response @app.exception_handler(VMError) async def handle_vm_error(_request: Request, exc: VMError): content = {"error": str(exc)} if exc.error_code: content["error_code"] = exc.error_code return JSONResponse(status_code=exc.status, content=content) @app.exception_handler(PublicIpClaimCleanupPending) async def handle_public_ip_claim_cleanup( _request: Request, exc: PublicIpClaimCleanupPending, ): return JSONResponse(status_code=exc.status, content={ "error": str(exc), "error_code": "public_ip_claim_cleanup_pending", }) @app.exception_handler(RemoteHostError) async def handle_remote_error(_request: Request, exc: RemoteHostError): content = {"error": str(exc), "error_code": exc.error_code} if exc.status == 423: content["error_code"] = "key_locked" return JSONResponse(status_code=exc.status, content=content) @app.exception_handler(SetupError) async def handle_setup_error(_request: Request, exc: SetupError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(DatabaseSetupError) async def handle_database_setup_error(_request: Request, exc: DatabaseSetupError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(DatabaseError) async def handle_database_error(_request: Request, exc: DatabaseError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(UpdateError) async def handle_update_error(_request: Request, exc: UpdateError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(TemplateError) async def handle_template_error(_request: Request, exc: TemplateError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(UserError) async def handle_user_error(_request: Request, exc: UserError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(SMTPError) async def handle_smtp_error(_request: Request, exc: SMTPError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) @app.exception_handler(SSHKeyError) async def handle_ssh_key_error(_request: Request, exc: SSHKeyError): return JSONResponse(status_code=exc.status, content={"error": str(exc)}) def account_for_session(session: Session) -> dict: if internal_mode: return actor_account(session) if database_mode: if state is None: raise HTTPException(status_code=401, detail="请先登录") user_id = getattr(session, "user_id", None) if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1: raise HTTPException(status_code=401, detail="账号不可用,请重新登录") account = state.users.get_by_id(user_id) if (account is None or account["username"] != session.username or not account["email_verified"]): raise HTTPException(status_code=401, detail="账号不可用,请重新登录") return account # The injected JSON state is retained solely for the isolated HTTP # test harness; production authorization always uses the users table. return {"id": 0, "username": session.username, "admin": True, "approved": True, "email_verified": True} def host_ready() -> bool: if manager is None: return False profile = manager.current() return profile is not None and not profile.get("key_locked", False) def require_session(request: Request) -> Session: if internal_mode: return actor_session() session = sessions.get(request.cookies.get(COOKIE_NAME)) if session is None: raise HTTPException(status_code=401, detail="请先登录") account_for_session(session) return session def require_write(request: Request) -> Session: session = require_session(request) if internal_mode: if not actor()["can_manage"]: raise HTTPException(status_code=403, detail="当前账户不能执行虚拟机操作") return session csrf = request.headers.get("X-CSRF-Token", "") if not hmac.compare_digest(csrf, session.csrf_token): raise HTTPException(status_code=403, detail="请求验证失败") return session def require_admin(request: Request) -> Session: session = require_session(request) if not account_for_session(session)["admin"]: raise HTTPException(status_code=403, detail="仅管理员可以执行此操作") return session def require_admin_write(request: Request) -> Session: session = require_write(request) if internal_mode and not actor()["is_super_admin"]: raise HTTPException(status_code=403, detail="只有超级管理员可以修改宿主机或镜像") if not account_for_session(session)["admin"]: raise HTTPException(status_code=403, detail="仅管理员可以执行此操作") return session def require_approved(request: Request) -> Session: session = require_session(request) account = account_for_session(session) if not account["admin"] and not account["approved"]: raise HTTPException(status_code=403, detail="账号等待管理员批准") return session def require_approved_write(request: Request) -> Session: session = require_write(request) account = account_for_session(session) if not account["admin"] and not account["approved"]: raise HTTPException(status_code=403, detail="账号等待管理员批准") return session def require_internal_service_write(request: Request) -> Session: if not internal_mode: raise HTTPException(status_code=404, detail="Not Found") return require_approved_write(request) def require_vm_access(session: Session, profile: dict, vm_id: str) -> str: if not database_mode: return vm_id try: canonical_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc account = account_for_session(session) if not account["admin"] and not state.users.owns_vm( host_scope(profile), canonical_id, account["id"]): raise VMError("虚拟机不存在", 404) return canonical_id def smtp_store() -> SMTPSettingsStore: if data_store is None: raise SMTPError("请先完成数据库配置", 409) return SMTPSettingsStore(data_store) def ssh_key_store(): if not database_mode or state is None: raise SSHKeyError("请先完成数据库配置", 409) return state.ssh_keys def current_updater() -> Updater: nonlocal updater if data_store is None: raise UpdateError("请先完成数据库配置", 409) if updater is None: updater = Updater(settings.static_dir.parent, data_store) return updater def forbid_long_job_during_update() -> None: if data_store is None: return active_id = data_store.get_value("update.active_job") if active_id: job = current_updater().get_job(active_id) if job is not None and job.get("status") in {"queued", "running"}: raise HTTPException(status_code=409, detail="面板正在更新,暂时不能开始新任务") def current_profile() -> dict: profile = manager.current() if profile is None: raise HTTPException(status_code=409, detail="请先在面板中配置宿主机") if profile.get("key_locked"): raise RemoteHostError("宿主机 SSH 私钥已锁定,请先在面板解锁", 423) return profile def new_service(profile: dict) -> VMService: if service_factory is not None: return service_factory(profile) return VMService( profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"], profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""), ) def verified_created_vm(profile: dict, result: dict, requested_uuid: str) -> dict: if result.get("uuid") == requested_uuid: return result # Keep the preallocated owner/credential records if libvirt may have # created the requested guest. Never delete a returned, mismatched UUID: # it may refer to a different existing VM. LOG.error("VM create returned a mismatched UUID: expected=%s", requested_uuid) try: verified = new_service(profile).get_vm(requested_uuid) except VMError: verified = None if verified is not None and verified.get("uuid") == requested_uuid: LOG.warning("Recovered VM result by querying the requested UUID: %s", requested_uuid) return verified raise VMError("虚拟机创建结果与预分配 UUID 不一致;请由管理员核查宿主机及归属记录", 502) def current_service() -> VMService: return new_service(current_profile()) def validate_staged_libvirt( staged, body: HostBody, *, ensure_paths: bool = False, ) -> tuple[dict | None, str | None]: service = new_service({ "libvirt_uri": staged.libvirt_uri, "storage_pool": body.storage_pool, "iso_pool": body.iso_pool, "rescue_iso_pool": body.rescue_iso_pool, "default_network": body.default_network, }) # Validate only host-owned storage/network resources here. Host save # and connection testing must never enumerate every guest domain; # capacity and IP scheduling are panel-owned and VM operations use # targeted UUID reads when they need a particular guest. if ensure_paths and (body.rescue_iso_pool or any(path.startswith("/") for path in ( body.storage_pool, body.iso_pool))): service.ensure_storage_pools() try: return service.resources(), None except VMError as exc: return None, str(exc) def session_response(username: str) -> JSONResponse: if database_mode: account = state.users.get_by_username(username) if account is None: raise HTTPException(status_code=401, detail="账号不可用,请重新登录") session = sessions.issue(username, user_id=account["id"]) else: session = sessions.issue(username) account = account_for_session(session) response = JSONResponse({ "authenticated": True, "username": session.username, "csrf_token": session.csrf_token, "admin": account["admin"], "approved": account["approved"], "host_ready": host_ready(), }) response.set_cookie( COOKIE_NAME, session.token, max_age=8 * 60 * 60, httponly=True, secure=settings.secure_cookie, samesite="strict", path="/", ) return response @app.get("/", include_in_schema=False) def index(): return FileResponse(settings.static_dir / "index.html") @app.get("/api/v1/health") def health(): if data_store is not None: # A running HTTP listener alone is not a usable post-update panel. # The updater treats this endpoint as readiness for the configured DB. if database_mode: state.users.has_users() else: data_store.get("admin", "singleton") return {"status": "ok"} @app.get("/api/v1/database/status") def database_status(): if bootstrap is None: return {"configured": True, "setup_required": False} return {"configured": bootstrap.configured(), "setup_required": not bootstrap.configured()} @app.post("/api/v1/database/setup") def setup_database(body: DatabaseSetupBody, request: Request): nonlocal state, manager, templates, data_store, updater, sessions, rescue_catalog nonlocal user_iso_catalog if bootstrap is None: raise DatabaseSetupError("数据库初始化仅供正常部署使用", 409) peer = _peer(request) if not setup_limiter.allowed(peer): raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试") def connect(config: dict, master_key: bytes): try: return make_components( settings.data_dir, config, master_key, settings.ssh_binary) except Exception as exc: # Connector errors may include database host details. Never # reflect the exception, credentials or driver SQL in HTTP. LOG.warning("database setup connection failed: %s", type(exc).__name__) raise DatabaseSetupError( "无法连接或初始化 MariaDB;请检查地址、账号、数据库和建表权限", 502, ) from exc try: components = bootstrap.configure({ "host": body.host, "port": body.port, "database": body.database, "username": body.username, "password": body.password.get_secret_value(), }, body.setup_token, connect) except DatabaseSetupError: setup_limiter.failed(peer) raise state, manager, templates, data_store = components rescue_catalog = RescueTemplateStore(data_store) user_iso_catalog = UserISOStore(data_store) sessions = DatabaseSessionStore(data_store) restore_jobs() setup_limiter.succeeded(peer) LOG.info("panel database configured") return {"configured": True} @app.get("/api/v1/setup/status") def setup_status(): if state is None: raise HTTPException(status_code=409, detail="请先配置数据库") return {"setup_required": state.setup_required()} @app.post("/api/v1/setup/admin") def setup_admin(body: SetupBody, request: Request): if state is None: raise HTTPException(status_code=409, detail="请先配置数据库") peer = _peer(request) if not setup_limiter.allowed(peer): raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试") try: username = state.create_admin(body.token, body.username, body.password) except SetupError: setup_limiter.failed(peer) raise setup_limiter.succeeded(peer) LOG.info("panel initial user created: username=%s", username) return session_response(username) @app.get("/api/v1/session") def session_info(request: Request): session = sessions.get(request.cookies.get(COOKIE_NAME)) if session is None: return {"authenticated": False, "username": None, "csrf_token": None} try: account = account_for_session(session) except HTTPException: return {"authenticated": False, "username": None, "csrf_token": None} return { "authenticated": True, "username": session.username, "csrf_token": session.csrf_token, "admin": account["admin"], "approved": account["approved"], "host_ready": host_ready(), } @app.post("/api/v1/login") def login(body: LoginBody, request: Request): if state is None: raise HTTPException(status_code=409, detail="请先配置数据库") peer = _peer(request) if not login_limiter.allowed(peer): raise HTTPException(status_code=429, detail="登录尝试过多,请稍后再试") if database_mode: if body.email is None or body.username is not None: raise HTTPException(status_code=422, detail="请使用邮箱登录") try: email = normalize_email(body.email) except UserError: login_limiter.failed(peer) raise HTTPException(status_code=401, detail="邮箱或密码错误") from None if not state.users.has_users(): raise HTTPException(status_code=409, detail="请先完成初始化") account = state.users.get_by_email(email) encoded = account["password_hash"] if account else None username = account["username"] if account else "" identity_ok = account is not None else: legacy_admin = state.admin() if legacy_admin is None: raise HTTPException(status_code=409, detail="请先完成初始化") username, encoded = legacy_admin account = {"email_verified": True} identity_ok = hmac.compare_digest(body.username or "", username) password_ok = verify_password(body.password, encoded) if encoded else False if not password_ok or not identity_ok or not account["email_verified"]: login_limiter.failed(peer) raise HTTPException(status_code=401, detail="邮箱或密码错误") login_limiter.succeeded(peer) return session_response(username) def deliver_registration_code(config, email: str, code: str, users) -> None: try: send_verification_code(config, email, code) except Exception as exc: # Background delivery must not expose whether an email exists or # relay SMTP server diagnostics to an unauthenticated client. LOG.warning("Registration email delivery failed: %s", type(exc).__name__) try: users.revoke_registration_code(email, code) except DatabaseError: LOG.exception("Could not revoke undelivered registration code") @app.post("/api/v1/register/code") def send_registration_code(body: RegistrationCodeBody, request: Request, background_tasks: BackgroundTasks): if not database_mode or state is None or not state.users.has_users(): raise HTTPException(status_code=409, detail="请先完成面板初始化") peer = _peer(request) if not code_send_limiter.allowed(peer): raise HTTPException(status_code=429, detail="验证码请求过多,请稍后再试") email = normalize_email(body.email) config = smtp_store().get() if config is None: raise SMTPError("管理员尚未配置 SMTP,暂不能注册", 503) code = state.users.generate_code() try: should_send = state.users.issue_registration_code(email, code) except UserError as exc: if exc.status != 429: raise should_send = False code_send_limiter.failed(peer) if should_send: # Sending after the HTTP response avoids exposing whether an # address is registered via SMTP latency or transport failures. background_tasks.add_task( deliver_registration_code, config, email, code, state.users, ) # Identical public response for existing addresses and cooldowns. return {"sent": True} @app.post("/api/v1/register", status_code=201) def register(body: RegistrationBody, request: Request): if not database_mode or state is None or not state.users.has_users(): raise HTTPException(status_code=409, detail="请先完成面板初始化") peer = _peer(request) if not registration_limiter.allowed(peer): raise HTTPException(status_code=429, detail="注册尝试过多,请稍后再试") try: state.users.register_with_code(body.username, body.email, body.password, body.code) except UserError: registration_limiter.failed(peer) raise registration_limiter.succeeded(peer) return {"registered": True, "approved": False} @app.post("/api/v1/logout") def logout(request: Request, _session: Session = Depends(require_write)): sessions.revoke(request.cookies.get(COOKIE_NAME)) response = JSONResponse({"ok": True}) response.delete_cookie(COOKIE_NAME, path="/") return response @app.get("/api/v1/ssh-keys") def list_ssh_keys(_session: Session = Depends(require_session)): account = account_for_session(_session) return {"items": ssh_key_store().list_for_user(account["id"])} @app.post("/api/v1/ssh-keys", status_code=201) def create_ssh_key(body: SSHKeyBody, _session: Session = Depends(require_write)): account = account_for_session(_session) return ssh_key_store().create_for_user( account["id"], body.name, body.public_key, ) @app.delete("/api/v1/ssh-keys/{key_id}") def delete_ssh_key(key_id: int, _session: Session = Depends(require_write)): if key_id < 1: raise SSHKeyError("公钥不存在", 404) account = account_for_session(_session) if not ssh_key_store().delete_for_user(account["id"], key_id): raise SSHKeyError("公钥不存在", 404) return {"deleted": True} @app.get("/api/v1/smtp/settings") def get_smtp_settings(_session: Session = Depends(require_admin)): return smtp_store().public_status() @app.put("/api/v1/smtp/settings") def save_smtp_settings(body: SMTPSettingsBody, _session: Session = Depends(require_admin_write)): settings_data = body.model_dump(exclude={"password"}) settings_data["password"] = (body.password.get_secret_value() if body.password is not None else "") config = smtp_store().save(settings_data) LOG.info("SMTP settings changed: admin=%s", _session.username) return config.public_status() @app.get("/api/v1/users/pending") def pending_users(_session: Session = Depends(require_admin)): if not database_mode: return {"items": []} return {"items": state.users.list_pending_users()} @app.post("/api/v1/users/{user_id}/approve") def approve_user(user_id: int, _session: Session = Depends(require_admin_write)): if not database_mode: raise HTTPException(status_code=409, detail="数据库用户功能不可用") account = state.users.approve_user(user_id) if account is None: raise HTTPException(status_code=404, detail="用户不存在") LOG.info("user approved: id=%s admin=%s", user_id, _session.username) return {"approved": True} def public_update_settings() -> dict: info = current_updater().settings() return { "repository_url": info["repository"], "branch": info["branch"], "token_configured": info["token_configured"], "checkout_ready": info["checkout_ready"], "configured": info["token_configured"] and info["checkout_ready"], "current_version": info["current_commit"], "active_job_id": info.get("active_job_id"), "last_job_id": info.get("last_job_id"), "checkout_error": info.get("checkout_error"), } @app.get("/api/v1/update/settings") def update_settings(_session: Session = Depends(require_admin)): return public_update_settings() @app.put("/api/v1/update/settings") def save_update_settings(body: UpdateSettingsBody, _session: Session = Depends(require_admin_write)): if body.repository_url != REPOSITORY_URL or body.branch != BRANCH: raise UpdateError("更新仓库和分支固定为面板指定的 Gitea main 分支", 422) service = current_updater() if body.access_token is not None: token = body.access_token.get_secret_value() if token: service.set_token(token) if not service.settings()["token_configured"]: raise UpdateError("请填写 Gitea 私有仓库访问令牌", 422) LOG.info("panel update access token configured: admin=%s", _session.username) return public_update_settings() @app.post("/api/v1/update/check") def check_update(_session: Session = Depends(require_admin_write)): result = current_updater().check() return { "repository_url": result["repository"], "branch": result["branch"], "current_version": result["current_commit"], "latest_version": result["remote_commit"], "update_available": result["status"] == "update_available", } @app.post("/api/v1/update/apply", status_code=202) def apply_update(body: UpdateApplyBody, _session: Session = Depends(require_admin_write)): with (import_lock, reinstall_lock, rescue_lock, snapshot_lock, rescue_import_lock): if (pending_templates or pending_reinstalls or pending_rescues or pending_snapshot_jobs or pending_rescue_imports): raise UpdateError("镜像导入或虚拟机任务正在执行,请等待完成后再更新面板", 409) try: unit = subprocess.run( ["/usr/bin/systemctl", "is-active", "virsh-panel-update.service"], capture_output=True, text=True, timeout=5, check=False, ) except (OSError, subprocess.TimeoutExpired): unit = None if unit is not None and unit.stdout.strip() in { "activating", "active", "deactivating", "reloading", }: raise UpdateError("独立更新服务仍在运行,请等待其完全结束后再提交", 409) service = current_updater() job = service.create_job( expected_current_commit=body.expected_current_commit, expected_remote_commit=body.expected_remote_commit, ) try: # The web service has a read-only code tree. A separate one-shot # unit performs the Git update and restarts this service. trigger = subprocess.run( ["/usr/bin/systemctl", "--no-block", "start", "virsh-panel-update.service"], capture_output=True, text=True, timeout=10, check=False, ) except (OSError, subprocess.TimeoutExpired): trigger = None if trigger is None or trigger.returncode != 0: service.fail_job(job["job_id"], "无法启动独立更新服务;请检查 virsh-panel-update.service 是否已安装") raise UpdateError("无法启动独立更新服务;请检查 systemd 配置", 503) LOG.info("panel update queued: job=%s admin=%s", job["job_id"], _session.username) return {"job_id": job["job_id"], "status": "queued"} @app.get("/api/v1/update/jobs/{job_id}") def update_job(job_id: str, _session: Session = Depends(require_admin)): job = current_updater().get_job(job_id) if job is None: raise UpdateError("更新任务不存在", 404) return job @app.get("/api/v1/host") def host(_session: Session = Depends(require_admin)): return _public_host(manager.current()) @app.get("/api/v1/host/network-pools") def host_network_pools(_session: Session = Depends(require_admin)): profile = current_profile() pools, compatibility_mode = public_network_pools(profile) return { "items": pools, "count": len(pools), "maximum": MAX_PUBLIC_NETWORK_POOLS, "compatibility_mode": compatibility_mode, } @app.put("/api/v1/host/network-pools/{pool_key}") def save_host_network_pool( pool_key: str, body: PublicNetworkPoolBody, _session: Session = Depends(require_admin_write), ): # Host saves and pool mutations share this lock so a concurrent bridge # change cannot commit between catalog validation and installation. # Production runs one engine worker; the catalog CAS still protects # independent pool writers and test/in-process concurrency. with public_pool_catalog_lock: profile = current_profile() pool, idempotent = upsert_public_network_pool(profile, pool_key, body) LOG.info( "host public network pool saved: host=%s pool=%s revision=%s admin=%s", profile.get("host"), pool_key, pool["revision"], _session.username, ) return { "pool": pool, "applied": not idempotent, "idempotent": idempotent, "compatibility_mode": False, } @app.post("/api/v1/host/scan") def scan_host(body: HostScanBody, _session: Session = Depends(require_admin_write)): return manager.scan(body.host, body.port) @app.post("/api/v1/host/unlock") def unlock_host(body: UnlockBody, request: Request, _session: Session = Depends(require_admin_write)): peer = _peer(request) if not unlock_limiter.allowed(peer): raise HTTPException(status_code=429, detail="私钥口令尝试过多,请稍后再试") try: profile = manager.unlock_current(body.passphrase) except RemoteHostError as exc: if exc.status == 400: unlock_limiter.failed(peer) raise unlock_limiter.succeeded(peer) LOG.info("remote host SSH key unlocked: host=%s admin=%s", profile.get("host"), _session.username) return _public_host(profile) @app.post("/api/v1/host/test") def test_host(body: HostBody, _session: Session = Depends(require_admin_write)): staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) try: result = manager.test(staged) resources, resource_warning = validate_staged_libvirt(staged, body) return { **result, "resources": resources, "warnings": [resource_warning] if resource_warning else [], } finally: manager.abort(staged) @app.put("/api/v1/host") def save_host(body: HostBody, _session: Session = Depends(require_admin_write)): # A running reinstall uses the saved SSH host identity and paths. Do # not replace them while its libvirt operation is in flight. with (reinstall_lock, rescue_lock, snapshot_lock, rescue_import_lock, public_pool_catalog_lock): forbid_long_job_during_update() if (pending_reinstalls or pending_rescues or pending_snapshot_jobs or pending_rescue_imports or any(job.get("status") in {"queued", "running", "uncertain"} for job in reinstall_jobs.values()) or (data_store is not None and data_store.list("vm_rescue"))): raise VMError("请先完成救援或重装任务并退出救援模式,再修改宿主机配置", 409) if any( job.get("status") in {"succeeded", "uncertain"} and "generated_password" in job for job in reinstall_jobs.values()): raise VMError("请先完成重装任务并确认保存自动生成的密码,再修改宿主机配置", 409) current = manager.current() if current is not None: remote_identity_changed = ( current.get("host") != body.host.lower() or current.get("port") != body.port or current.get("fingerprint") != body.confirmed_fingerprint or current.get("storage_pool") != body.storage_pool ) if remote_identity_changed and has_retained_snapshots(): raise VMError( "宿主机仍保留本地快照;更换远端宿主身份或存储池前请先删除全部快照", 409, ) _raw_catalog, native_pools = read_native_public_network_pools(current) if any( pool["network_selector"] != body.default_network for pool in native_pools): raise VMError( "宿主机已有公网地址池;请先在公网地址池管理中处理网络接入,再修改默认网络", 409, ) staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) committed = False try: manager.test(staged) _resources, resource_warning = validate_staged_libvirt(staged, body, ensure_paths=True) profile = manager.commit(staged) committed = True finally: if not committed: manager.abort(staged) LOG.info("remote host configured: host=%s port=%s admin=%s", body.host, body.port, _session.username) return { **_public_host(profile), "warnings": [resource_warning] if resource_warning else [], } @app.get("/api/v1/resources") def resources(_session: Session = Depends(require_approved)): profile = current_profile() values = new_service(profile).resources() # Root-pool ISO filenames are retained only for legacy/admin clients. # Ordinary users select owner-scoped opaque IDs from /user-isos. if database_mode and not account_for_session(_session)["admin"]: values["isos"] = [] pools, compatibility_mode = public_network_pools(profile) primary = next( (pool for pool in pools if pool["state"] == "active"), None) return { **values, # Keep the original scalar fields during the PHP rollout. They # describe only the deterministic primary active pool; new code # uses host/network-pools for the configured catalog. "public_pool_cidr": "" if primary is None else primary["cidr"], "public_pool_enabled": primary is not None, "public_vm_prefix": ( None if primary is None else primary["guest_prefix_length"]), "public_network_pool_count": len(pools), "public_network_pool_compatibility_mode": compatibility_mode, } def cloud_template_record(profile: dict, template_id: str) -> dict: if shared_cloud_template_runtime is not None: try: return shared_cloud_template_runtime.get(template_id) except TemplateError as exc: if exc.status != 404: raise # Host-scoped template IDs were used before the catalog became # global. Keep already-paid/queued operations executable on # their original host while administrators re-import those # images into the new global catalog. They cannot be promoted # automatically because the old flow intentionally discarded # the verified local source after uploading it to libvirt. return templates.get(template_scope(profile), template_id) return templates.get(template_scope(profile), template_id) def cloud_template_replica_scope(profile: dict) -> str: """Bind a ready marker to the actual remote host configuration. ``template_scope`` deliberately keeps a stable logical host identity. That is useful for legacy catalogs, but is too weak as proof that a cached volume still belongs to the same physical hypervisor. A host rebind or SSH host-key change must invalidate every ready marker. """ identity = [ "virt-cloud-template-replica-v2", profile.get("host_id"), profile.get("host"), profile.get("port"), profile.get("fingerprint"), profile.get("storage_pool"), ] return hashlib.sha256(json.dumps( identity, ensure_ascii=False, separators=(",", ":"), ).encode("utf-8")).hexdigest() def cloud_template_replica_volume_name(record: dict) -> str: source_name = record["volume_name"] if not isinstance(source_name, str) or not source_name.endswith(".qcow2"): raise DatabaseError("云镜像源卷名无效") return f"{source_name[:-6]}-{secrets.token_hex(8)}.qcow2" def valid_cloud_template_replica_volume_name( record: dict, volume_name: object) -> bool: source_name = record.get("volume_name") if (not isinstance(source_name, str) or not source_name.endswith(".qcow2") or not isinstance(volume_name, str)): return False prefix = re.escape(source_name[:-6]) return re.fullmatch( rf"{prefix}-[0-9a-f]{{16}}\.qcow2", volume_name, ) is not None def cloud_template_replica_payload( profile: dict, record: dict, status: Literal["syncing", "ready"], replica_volume_name: str, replica_identity: str | None = None) -> dict: if not valid_cloud_template_replica_volume_name( record, replica_volume_name): raise DatabaseError("云镜像宿主机缓存卷名无效") result = { "status": status, "replica_scope": cloud_template_replica_scope(profile), "source_volume_name": record["volume_name"], "replica_volume_name": replica_volume_name, "sha256": record["sha256"], "virtual_size": record["virtual_size"], "image_bytes": record["image_bytes"], } if status == "ready": if (not isinstance(replica_identity, str) or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None): raise DatabaseError("云镜像宿主机缓存身份无效") result["replica_identity"] = replica_identity return result def cloud_template_replica_marker( profile: dict, record: dict) -> dict | None: if data_store is None: return None raw = data_store.get("cloud_template_replica", record["id"]) if raw is None: return None try: value = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("云镜像宿主机缓存记录损坏") from exc if not isinstance(value, dict): raise DatabaseError("云镜像宿主机缓存记录损坏") status = value.get("status") replica_volume_name = value.get("replica_volume_name") if (status not in {"syncing", "ready"} or not valid_cloud_template_replica_volume_name( record, replica_volume_name)): # Older/incomplete marker schemas are evidence only that a cache # may exist, never proof that it is complete or belongs to this # physical host generation. return None replica_identity = value.get("replica_identity") if status == "ready" and ( not isinstance(replica_identity, str) or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None): return None expected = cloud_template_replica_payload( profile, record, status, replica_volume_name, replica_identity if status == "ready" else None, ) return value if value == expected else None def save_cloud_template_replica( profile: dict, record: dict, status: Literal["syncing", "ready"], replica_volume_name: str, replica_identity: str | None = None) -> None: if data_store is None: return data_store.set( "cloud_template_replica", record["id"], json.dumps( cloud_template_replica_payload( profile, record, status, replica_volume_name, replica_identity), separators=(",", ":"), ), ) def ensure_cloud_template_replica( profile: dict, service: VMService, template_id: str) -> dict: record = cloud_template_record(profile, template_id) if shared_cloud_template_runtime is None: return record with cloud_template_cache_lock: # Re-read after entering the per-host critical section. Another # create may have completed this same first-use transfer while the # request was waiting for the lock. try: record = shared_cloud_template_runtime.get(template_id) except TemplateError as exc: if exc.status != 404: raise return templates.get(template_scope(profile), template_id) marker = cloud_template_replica_marker(profile, record) trusted_volume_name = None trusted_identity = None if marker is not None and marker["status"] == "ready": trusted_volume_name = marker["replica_volume_name"] trusted_identity = marker["replica_identity"] # Persist a different, never-before-used fallback generation # before asking libvirt whether the ready volume is still the # exact volume proven by the marker. If it is not, the old # name is never overwritten or silently reused. desired_volume_name = cloud_template_replica_volume_name(record) elif marker is not None: # A crash after writing ``syncing`` may have left a partial # upload. Retry that planned generation as untrusted; the # service safely replaces it only when no VM references it. desired_volume_name = marker["replica_volume_name"] else: desired_volume_name = cloud_template_replica_volume_name(record) # Keep a proven ready marker until the global source has itself # been verified. A damaged source must not downgrade evidence of # an already complete host replica. with shared_cloud_template_runtime.source(template_id) as ( record, source_path): # Once source verification succeeds, always downgrade the # durable marker before touching libvirt. Even a previously # ready replica may have disappeared or need replacement; a # power loss during that upload must never leave ready proof # pointing at a partially streamed qcow2 volume. save_cloud_template_replica( profile, record, "syncing", desired_volume_name) cached = service.cache_template( source_path, desired_volume_name, record["virtual_size"], trusted_replica_volume_name=trusted_volume_name, trusted_replica_identity=trusted_identity, ) cached_volume_name = cached.get("volume_name") accepted_names = {desired_volume_name} if trusted_volume_name is not None: accepted_names.add(trusted_volume_name) if cached_volume_name not in accepted_names: raise DatabaseError("云镜像宿主机缓存返回了无效卷") save_cloud_template_replica( profile, record, "ready", cached_volume_name, cached.get("replica_identity")) return {**record, "volume_name": cached_volume_name} def user_iso_context(session: Session) -> tuple[int, str]: owner_id = account_for_session(session)["id"] return owner_id, user_iso_scope(owner_id) def user_iso_replica_key(owner_id: int, iso_id: str) -> str: return f"{owner_id}:{iso_id}" def trusted_user_iso_replica(owner_id: int, record: dict) -> bool: if data_store is None: return False raw = data_store.get( "user_iso_replica", user_iso_replica_key(owner_id, record["id"])) if raw is None: return False try: replica = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("用户 ISO 宿主机缓存记录损坏") from exc if not isinstance(replica, dict): raise DatabaseError("用户 ISO 宿主机缓存记录损坏") return replica == { "status": "ready", "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], } def save_user_iso_replica(owner_id: int, record: dict, *, status: Literal["syncing", "ready"]) -> None: if data_store is None: return data_store.set( "user_iso_replica", user_iso_replica_key(owner_id, record["id"]), json.dumps({ "status": status, "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], }, separators=(",", ":")), ) def prune_user_iso_replica_records(owner_id: int, active_ids: set[str]) -> None: if data_store is None: return prefix = f"{owner_id}:" for key in data_store.list("user_iso_replica"): if key.startswith(prefix) and key[len(prefix):] not in active_ids: data_store.delete("user_iso_replica", key) def public_user_iso(record: dict) -> dict: return {key: value for key, value in record.items() if key in {"id", "name", "size_bytes", "sha256", "created_at"}} def public_user_iso_error(value: Any) -> str: safe = { "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交", "ISO 下载地址无效", "ISO 下载地址无法解析", "ISO 下载地址的文件名必须以 .iso 结尾", "ISO 下载地址解析结果无效", "ISO 下载地址不能指向内网、回环或保留地址", "ISO 下载地址发生重定向,请使用最终 HTTPS 直链", "无法连接 ISO 下载地址", "ISO 下载响应长度无效", "ISO 下载响应不能使用内容压缩", "ISO 下载超时", "ISO 下载内容不完整", "ISO 的 SHA-256 校验值不匹配", "ISO 大小必须在 1 MiB 至可用配额之间", "下载文件不是有效的 ISO 9660 镜像", "ISO 未包含可识别的 El Torito 启动记录", "ISO 超过用户目录的 10 GiB 总配额或剩余容量", "用户 ISO 可用配额不足", "每个宿主机最多保存 5 个用户 ISO", "用户 ISO 目录的 10 GiB 存储配额已用完", "用户 ISO 目录总容量不能超过 10 GiB", "宿主机用户 ISO 存储空间不足", "用户 ISO 上传到宿主机失败", "用户 ISO 导入失败,请查看引擎日志", } if isinstance(value, str) and value in safe: return value if isinstance(value, str) and re.fullmatch( r"ISO 下载服务器返回 HTTP [1-5][0-9]{2}", value): return value return "ISO 下载或导入失败,请联系管理员查看日志" # Async job records are persisted by the engine and may contain text from # libvirt, QEMU, a remote host, or a future implementation. The browser # endpoints below deliberately expose a small, operation-specific DTO; # never return a job dictionary wholesale. Keep these sets local to the # app factory so an engine instance cannot accidentally share mutable # presentation state with another host. public_job_statuses = frozenset({ "queued", "pending", "running", "succeeded", "completed", "done", "failed", "uncertain", "cancelled", "canceled", "blocked", "unknown", }) public_iso_phases = frozenset({ "等待下载", "正在安全下载 ISO", "正在保存到面板 ISO 目录", "ISO 已保存,可用于所有宿主机", "导入失败", "引擎重启,任务已中断", "面板在 ISO 下载期间重启;请重新提交导入", }) public_reinstall_phases = frozenset({ "等待重装任务开始", "正在检查虚拟机", "重装完成", "重装失败", "重装完成,但旧系统盘清理待核对", "重装结果待核对", "重装已完成,但凭据或任务记录保存失败", "面板重启,任务已中断", # These are stable progress identifiers consumed by the standalone # engine UI; they are not arbitrary text from libvirt. "shutting_down", "cloning", "starting", "cleaning_up", "cleanup_pending", "result_uncertain", "completed", }) public_rescue_phases = frozenset({ "等待救援任务开始", "等待手动确认的救援冷启动", "等待退出救援模式", "正在预挂载救援 ISO 并等待正常关机", "正在强制断电并从救援 ISO 冷启动", "正在正常关机并恢复原系统启动配置", "救援 ISO 已预挂载,等待手动强制断电后开机", "救援系统已启动", "已退出救援模式,原系统已启动", "救援操作失败", "面板重启,任务已中断", }) # Canonical messages retain useful, already-reviewed user guidance while # removing any suffix supplied by a host-controlled exception. Prefixes # are matched only to select a fixed replacement; the original value is # never returned. public_reinstall_errors = ( ("重装失败且自动回滚不完整", "重装失败且自动回滚不完整;旧系统盘已保留,请检查虚拟机与新建存储卷"), ("重装失败,已恢复原虚拟机定义及旧系统盘", "重装失败,已恢复原虚拟机定义及旧系统盘"), ("正常关机超时,重装已停止", "正常关机超时,重装已停止;旧系统盘未改变"), ("重装系统失败", "重装系统失败,请查看面板日志"), ("虚拟机配置无效", "虚拟机配置无效,重装已停止"), ("虚拟机磁盘或网卡配置已变化", "虚拟机配置已变化,重装已停止"), ("虚拟机网桥或 MAC 已变化", "虚拟机网桥或 MAC 已变化,不能自动沿用原网络"), ("关机后虚拟机配置已变化", "关机后虚拟机配置已变化,重装已停止"), ("关机后存储卷已变化", "关机后存储卷已变化,重装已停止"), ("原配置盘的公网 IP 或 MAC 与虚拟机不一致", "原配置盘的公网 IP 或 MAC 与虚拟机不一致,已停止重装"), ("无法校验原 cloud-init 网络配置", "无法校验原 cloud-init 网络配置,已停止重装"), ("模板卷在宿主机上不存在", "模板卷在宿主机上不存在,请重新导入"), ("模板卷不是 qcow2 格式", "模板卷格式不受支持,已停止重装"), ("模板容量超过原系统盘", "所选模板容量超过原系统盘,无法保留原磁盘大小"), ("存储池空间不足", "存储池空间不足,无法创建新系统盘"), ("新系统盘或配置盘名称冲突", "新系统盘或配置盘名称冲突,已停止重装"), ("宿主机未能克隆新系统盘", "宿主机未能克隆新系统盘,重装已停止"), ("宿主机未能创建新配置盘", "宿主机未能创建新配置盘,重装已停止"), ("宿主机未能切换虚拟机定义", "宿主机未能切换虚拟机定义,重装已停止"), ("宿主机未能保存虚拟机网络配置", "宿主机未能保存新的虚拟机配置,重装已停止"), ("重装已完成,但旧系统盘清理待核对", "重装已完成,但旧系统盘清理待核对,请管理员检查宿主机"), ("重装结果待核对", "重装结果待核对,请检查虚拟机状态和存储卷"), ) public_rescue_errors = ( ("救援 ISO 预挂载失败且原虚拟机持久配置未能恢复", "救援 ISO 预挂载失败且原虚拟机持久配置未能恢复;没有强制断电,请检查宿主机"), ("救援 ISO 预挂载失败", "救援 ISO 预挂载失败,原虚拟机未受影响"), ("救援启动失败,已恢复原虚拟机配置及运行状态", "救援启动失败,已恢复原虚拟机配置及运行状态"), ("救援系统强制重启后未能启动", "救援系统强制重启后未能启动;救援 ISO 配置已保留,请检查控制台"), ("已尝试强制启动,但救援系统未进入运行状态", "已尝试强制启动,但救援系统未进入运行状态;救援 ISO 配置已保留,请检查控制台或重试开机"), ("退出救援失败且救援虚拟机未能完全恢复", "退出救援失败且救援虚拟机未能完全恢复;没有强制断电,请检查宿主机"), ("退出救援失败,已恢复救援启动配置及运行状态", "退出救援失败,已恢复救援启动配置及运行状态"), ("原系统已启动但仍显示救援模式", "原系统已启动但仍显示救援模式,请检查虚拟机配置"), ("关机后救援启动配置已变化", "关机后救援启动配置已变化,救援已停止;请检查虚拟机状态"), ("强制断电失败", "强制断电失败;救援配置已保留,请检查虚拟机状态"), ("救援 ISO 启动设备与预挂载记录不匹配", "救援 ISO 启动设备与预挂载记录不匹配,已拒绝强制重启"), ("救援恢复记录丢失", "救援恢复记录丢失,已停止操作"), ("救援操作失败", "紧急救援操作失败,请查看面板和宿主机日志"), ("正常关机超时", "正常关机超时,救援未完成,请核对救援状态后再继续"), ("无法请求虚拟机正常关机", "无法请求虚拟机正常关机,请核对救援状态"), ) public_reinstall_warnings = ( ("新系统可能已启动,但启动返回错误", "新系统可能已启动,但启动结果需要通过控制台确认;旧系统盘已保留"), ("新系统已启动,但面板未能读取完整虚拟机状态", "新系统已启动,但面板未能读取完整虚拟机状态,请检查宿主机"), ("新系统已启动,但面板无法确认初始密码已保存", "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码"), ("旧存储卷", "重装已完成,但旧系统盘清理需要管理员检查"), ) public_rescue_warnings = ( ("救援配置已运行,但宿主机关机状态返回异常", "救援配置已运行,但宿主机关机状态需要通过 VNC 确认"), ("救援系统可能已启动,但宿主机启动返回错误", "救援系统可能已启动,请通过控制台确认"), ("救援系统可能已启动,但宿主机返回错误", "救援系统可能已启动,请通过控制台确认"), ("原系统可能已启动,但宿主机启动返回错误", "原系统可能已启动,请通过控制台确认"), ("救援 ISO 已配置为下次冷启动", "救援 ISO 已配置为下次冷启动,请手动强制断电后开机"), ("正常关机超时", "正常关机超时;救援 ISO 可能已预挂载,请核对救援状态"), ("正常关机", "正常关机未完成;请核对救援状态后再继续"), ) def public_status(value: Any) -> str: return value if isinstance(value, str) and value in public_job_statuses else "unknown" def public_timestamp(value: Any) -> int | None: if isinstance(value, bool): return None try: result = int(value) except (TypeError, ValueError, OverflowError): return None return result if result >= 0 else None def public_identifier(value: Any, *, minimum: int = 1, maximum: int = 128) -> str | None: if not isinstance(value, str) or not (minimum <= len(value) <= maximum): return None return value if re.fullmatch(r"[A-Za-z0-9_-]+", value) else None def canonical_job_text(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None: if not isinstance(value, str) or not value.strip(): return None text = value.strip() for prefix, canonical in mappings: if text.startswith(prefix): return canonical return fallback def public_phase(value: Any, status: str, allowed: frozenset[str], *, kind: str) -> str: if kind == "reinstall" and status == "uncertain": if isinstance(value, str) and value in { "重装完成,但旧系统盘清理待核对", "重装结果待核对"}: return value return "等待核对结果" if isinstance(value, str) and value in allowed: return value if kind == "reinstall": return { "queued": "等待重装任务开始", "pending": "等待重装任务开始", "running": "正在执行重装任务", "succeeded": "重装完成", "completed": "重装完成", "done": "重装完成", "failed": "重装失败", "uncertain": "等待核对结果", }.get(status, "重装任务状态待核对") if kind == "rescue": return { "queued": "等待救援任务开始", "pending": "等待救援任务开始", "running": "正在执行救援任务", "succeeded": "救援操作已完成", "completed": "救援操作已完成", "done": "救援操作已完成", "failed": "救援操作失败", "uncertain": "等待核对结果", }.get(status, "救援任务状态待核对") return { "queued": "等待下载", "pending": "等待下载", "running": "正在处理", "succeeded": "已完成", "completed": "已完成", "done": "已完成", "failed": "导入失败", "uncertain": "等待核对结果", }.get(status, "任务状态待核对") def public_warning(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None: return canonical_job_text(value, mappings, fallback) def public_user_iso_job(job: dict) -> dict: status = public_status(job.get("status")) result: dict[str, Any] = {} job_id = public_identifier(job.get("job_id"), minimum=8) if job_id is not None: result["job_id"] = job_id result["status"] = status result["phase"] = public_phase( job.get("phase"), status, public_iso_phases, kind="iso") for key in ("created_at", "started_at", "finished_at"): timestamp = public_timestamp(job.get(key)) if timestamp is not None: result[key] = timestamp if isinstance(job.get("iso"), dict): result["iso"] = public_user_iso(job["iso"]) if "error" in job: result["error"] = public_user_iso_error(job.get("error")) return result def public_user_iso_listing(value: Any) -> dict: """Re-sanitize shared-runtime output at this app's HTTP boundary.""" if not isinstance(value, dict): return {"isos": []} result: dict[str, Any] = { "isos": [public_user_iso(item) for item in value.get("isos", []) if isinstance(item, dict)], } active = value.get("active_import_job") if isinstance(active, dict): result["active_import_job"] = public_user_iso_job(active) return result @app.get("/api/v1/user-isos") def list_user_isos(_session: Session = Depends(require_approved)): owner_id, scope = user_iso_context(_session) if shared_user_iso_runtime is not None: return public_user_iso_listing(shared_user_iso_runtime.list_for(owner_id)) result = { "isos": [public_user_iso(row) for row in user_iso_catalog.list(scope)], } with user_iso_import_lock: active = [job for job in user_iso_import_jobs.values() if job.get("scope") == scope and job.get("owner_user_id") == owner_id and job.get("status") in {"queued", "running"}] if active: # Only one import per owner/host can be submitted. Sorting is # defensive for an upgraded database that contains an older # duplicate; never expose either owner's private scope field. current = max(active, key=lambda job: ( int(job.get("created_at", 0)), str(job.get("job_id", "")))) result["active_import_job"] = public_user_iso_job(current) return result @app.get("/api/v1/user-isos/import-jobs/{job_id}") def user_iso_import_job(job_id: str, _session: Session = Depends(require_approved)): owner_id, scope = user_iso_context(_session) if shared_user_iso_runtime is not None: return public_user_iso_job( shared_user_iso_runtime.get_job_for(owner_id, job_id)) with user_iso_import_lock: job = user_iso_import_jobs.get(job_id) if (job is None or job.get("scope") != scope or job.get("owner_user_id") != owner_id): raise TemplateError("ISO 导入任务不存在", 404) return public_user_iso_job(job) @app.get("/api/v1/user-isos/{iso_id}") def get_user_iso(iso_id: str, _session: Session = Depends(require_approved)): owner_id, scope = user_iso_context(_session) if shared_user_iso_runtime is not None: return {"iso": public_user_iso( shared_user_iso_runtime.get_for(owner_id, iso_id))} return {"iso": public_user_iso(user_iso_catalog.get(scope, iso_id))} def run_user_iso_import(job_id: str, owner_id: int, scope: str, body: ImportUserISOBody) -> None: image_path: Path | None = None iso_id = "" published = False cataloged = False try: with user_iso_import_lock: job = user_iso_import_jobs[job_id] job.update(status="running", phase="正在安全下载 ISO", started_at=int(time.time())) save_user_iso_job(job_id, job) remaining = user_iso_catalog.remaining_bytes(scope) available = user_iso_files.available_bytes(owner_id) maximum = min(remaining, available) if maximum < 1024 * 1024: raise TemplateError("面板用户 ISO 持久目录空间不足", 409) image_path, size, digest = download_user_iso( body.url, body.sha256.lower(), user_iso_files.owner_directory(owner_id), max_bytes=maximum, ) iso_id = uuid.uuid4().hex record = { "id": iso_id, "name": body.name, "volume_name": user_iso_volume_name(iso_id), "size_bytes": size, "sha256": digest, "created_at": int(time.time()), } with user_iso_import_lock: job = user_iso_import_jobs[job_id] job["phase"] = "正在保存到面板 ISO 目录" save_user_iso_job(job_id, job) user_iso_files.publish( image_path, owner_id, iso_id, expected_size=size) image_path = None published = True user_iso_catalog.add(scope, record) cataloged = True user_iso_import_jobs[job_id] = { "job_id": job_id, "scope": scope, "owner_user_id": owner_id, "status": "completed", "phase": "ISO 已保存,可用于所有宿主机", "created_at": user_iso_import_jobs[job_id]["created_at"], "started_at": user_iso_import_jobs[job_id].get("started_at"), "finished_at": int(time.time()), "iso": record, } save_user_iso_job(job_id, user_iso_import_jobs[job_id]) except Exception as exc: if isinstance(exc, TemplateError): error = str(exc) else: LOG.exception("User ISO import failed: job=%s owner=%s", job_id, owner_id) error = "用户 ISO 导入失败,请查看引擎日志" if cataloged: try: user_iso_catalog.delete(scope, iso_id) except Exception: LOG.exception("Could not clean failed user ISO catalog: job=%s", job_id) if published and iso_id: try: user_iso_files.delete(owner_id, iso_id) except Exception: LOG.exception("Could not clean failed user ISO source: job=%s", job_id) with user_iso_import_lock: previous = user_iso_import_jobs.get(job_id, {}) user_iso_import_jobs[job_id] = { "job_id": job_id, "scope": scope, "owner_user_id": owner_id, "status": "failed", "phase": "导入失败", "error": error, "created_at": previous.get("created_at", int(time.time())), "started_at": previous.get("started_at"), "finished_at": int(time.time()), } try: save_user_iso_job(job_id, user_iso_import_jobs[job_id]) except DatabaseError: LOG.exception("Could not persist user ISO import failure: job=%s", job_id) finally: if image_path is not None: image_path.unlink(missing_ok=True) with user_iso_import_lock: pending_user_iso_imports.discard(scope) @app.post("/api/v1/user-isos/import", status_code=202) def import_user_iso(body: ImportUserISOBody, _session: Session = Depends(require_approved_write)): owner_id, scope = user_iso_context(_session) if shared_user_iso_runtime is not None: return shared_user_iso_runtime.queue_import( owner_id, body.url, body.name, body.sha256) validate_user_iso_url(body.url) if body.sha256 and (len(body.sha256) != 64 or not all(char in "0123456789abcdefABCDEF" for char in body.sha256)): raise TemplateError( "SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422) name = validate_user_iso_name(body.name) body = body.model_copy(update={"name": name, "sha256": body.sha256.lower()}) with user_iso_import_lock: forbid_long_job_during_update() existing = user_iso_catalog.list(scope) user_iso_catalog.remaining_bytes(scope) if any(row["name"].casefold() == name.casefold() for row in existing): raise TemplateError("当前用户已有同名 ISO", 409) if scope in pending_user_iso_imports: raise TemplateError("已有用户 ISO 正在导入,请等待完成", 409) job_id = str(uuid.uuid4()) pending_user_iso_imports.add(scope) user_iso_import_jobs[job_id] = { "job_id": job_id, "scope": scope, "owner_user_id": owner_id, "status": "queued", "phase": "等待下载", "created_at": int(time.time()), } try: # Persist before handing work to the executor. Both steps # share one rollback path so a database failure cannot leave # this owner permanently stuck in the in-memory pending set. save_user_iso_job(job_id, user_iso_import_jobs[job_id]) user_iso_import_executor.submit( run_user_iso_import, job_id, owner_id, scope, body) except Exception as exc: pending_user_iso_imports.discard(scope) user_iso_import_jobs.pop(job_id, None) with suppress(DatabaseError): delete_user_iso_job(job_id) if isinstance(exc, RuntimeError): raise TemplateError( "引擎正在停止,无法开始 ISO 导入", 503) from exc raise LOG.info("user ISO import queued: job=%s owner=%s", job_id, owner_id) return {"job_id": job_id, "status": "queued"} @app.delete("/api/v1/user-isos/{iso_id}") def delete_user_iso(iso_id: str, _session: Session = Depends(require_approved_write)): owner_id, scope = user_iso_context(_session) if shared_user_iso_runtime is not None: shared_user_iso_runtime.delete_for(owner_id, iso_id) LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id) return {"deleted": True} with user_iso_import_lock: forbid_long_job_during_update() if scope in pending_user_iso_imports: raise TemplateError("用户 ISO 正在导入,请稍后删除", 409) user_iso_catalog.get(scope, iso_id) user_iso_files.delete(owner_id, iso_id) user_iso_catalog.delete(scope, iso_id) LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id) return {"deleted": True} @app.get("/api/v1/vm-templates") def list_templates(_session: Session = Depends(require_approved)): if shared_cloud_template_runtime is not None: return {"templates": shared_cloud_template_runtime.list()} profile = current_profile() return {"templates": templates.list(template_scope(profile))} def run_template_import(job_id: str, profile: dict, scope: str, body: ImportTemplateBody, pending_key: str) -> None: image_path = None volume_name = "" uploaded = False try: image_path, virtual_size, actual_sha256 = download_verified_image( body.url, body.sha256.lower(), settings.data_dir) os_family, version = image_metadata_for(body.url) template_id = template_id_for( scope, actual_sha256, version, os_family ) volume_name = template_volume_name(template_id) with import_lock: if any(row["id"] == template_id for row in templates.list(scope)): raise TemplateError("此云镜像已经导入", 409) result = new_service(profile).import_template(image_path, volume_name, virtual_size) uploaded = True record = { "id": template_id, "name": body.name, "os_family": os_family, "os_version": version, "sha256": actual_sha256, "volume_name": volume_name, "virtual_size": virtual_size, "image_bytes": result["image_bytes"], } templates.add(scope, record) with import_lock: import_jobs[job_id] = {"scope": scope, "status": "completed", "template": record} save_job("import_job", job_id, import_jobs[job_id]) except Exception as exc: if isinstance(exc, (TemplateError, VMError)): error = str(exc) else: LOG.exception("Cloud template import failed: job=%s", job_id) error = "云镜像导入失败,请查看面板日志" if uploaded and volume_name: error += f";宿主机模板卷 {volume_name} 可能需要人工检查" with import_lock: import_jobs[job_id] = {"scope": scope, "status": "failed", "error": error} save_job("import_job", job_id, import_jobs[job_id]) finally: if image_path is not None: image_path.unlink(missing_ok=True) with import_lock: pending_templates.discard((scope, pending_key)) @app.post("/api/v1/vm-templates/import", status_code=202) def import_template(body: ImportTemplateBody, _session: Session = Depends(require_admin_write)): if shared_cloud_template_runtime is not None: forbid_long_job_during_update() return shared_cloud_template_runtime.queue_import( body.url, body.name, body.sha256) profile = current_profile() if body.sha256 and (len(body.sha256) != 64 or not all(char in "0123456789abcdefABCDEF" for char in body.sha256)): raise TemplateError( "SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422 ) os_family, version = image_metadata_for(body.url) name = validate_template_name(body.name) body = body.model_copy(update={"name": name}) scope = template_scope(profile) expected_sha256 = body.sha256.lower() pending_key = expected_sha256 or hashlib.sha256(body.url.encode("utf-8")).hexdigest() with import_lock: forbid_long_job_during_update() existing_templates = templates.list(scope) if expected_sha256 and any( row["id"] == template_id_for( scope, expected_sha256, version, os_family ) for row in existing_templates): raise TemplateError("此云镜像已经导入", 409) if any(row["name"].casefold() == name.casefold() for row in existing_templates): raise TemplateError("当前宿主机已有同名云镜像模板", 409) if (scope, pending_key) in pending_templates: raise TemplateError("该云镜像正在导入", 409) if pending_templates: raise TemplateError("已有云镜像正在导入,请等待完成后再提交", 409) job_id = str(uuid.uuid4()) pending_templates.add((scope, pending_key)) import_jobs[job_id] = {"scope": scope, "status": "running"} save_job("import_job", job_id, import_jobs[job_id]) try: import_executor.submit(run_template_import, job_id, profile, scope, body, pending_key) except RuntimeError as exc: pending_templates.discard((scope, pending_key)) import_jobs.pop(job_id, None) delete_job("import_job", job_id) raise TemplateError("面板正在停止,无法开始导入任务", 503) from exc LOG.info("cloud template import queued: job=%s admin=%s", job_id, _session.username) return {"job_id": job_id, "status": "running"} @app.patch("/api/v1/vm-templates/{template_id}") def rename_template(template_id: str, body: RenameTemplateBody, _session: Session = Depends(require_admin_write)): if shared_cloud_template_runtime is not None: forbid_long_job_during_update() return {"template": shared_cloud_template_runtime.rename( template_id, body.name)} scope = template_scope(current_profile()) with import_lock: if pending_templates: raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409) return {"template": templates.rename(scope, template_id, body.name)} @app.get("/api/v1/vm-templates/import-jobs/{job_id}") def import_job(job_id: str, _session: Session = Depends(require_admin)): if shared_cloud_template_runtime is not None: return shared_cloud_template_runtime.get_job(job_id) scope = template_scope(current_profile()) with import_lock: job = import_jobs.get(job_id) if job is None or job["scope"] != scope: raise TemplateError("导入任务不存在", 404) return {key: value for key, value in job.items() if key != "scope"} @app.get("/api/v1/rescue-templates") def list_rescue_templates(_session: Session = Depends(require_approved)): return {"templates": rescue_catalog.list(rescue_scope(current_profile()))} def run_rescue_import(job_id: str, profile: dict, scope: str, record: dict, image_path: Path) -> None: uploaded = False cataloged = False try: with rescue_import_lock: rescue_import_jobs[job_id]["status"] = "running" rescue_import_jobs[job_id]["phase"] = "正在传输 ISO 到宿主机" save_job("rescue_import_job", job_id, rescue_import_jobs[job_id]) new_service(profile).import_rescue_iso(image_path, record["volume_name"]) uploaded = True rescue_catalog.add(scope, record) cataloged = True with rescue_import_lock: rescue_import_jobs[job_id] = { "scope": scope, "status": "completed", "phase": "救援模版已导入", "template": record, } save_job("rescue_import_job", job_id, rescue_import_jobs[job_id]) except Exception as exc: if isinstance(exc, (TemplateError, VMError)): error = str(exc) else: LOG.exception("Rescue ISO import failed: job=%s", job_id) error = "救援 ISO 导入失败,请查看面板日志" if cataloged: try: rescue_catalog.delete(scope, record["id"]) except Exception: LOG.exception("Could not clean failed rescue catalog entry: job=%s", job_id) error += ";数据库中可能留有救援模版记录,请管理员检查" if uploaded: try: new_service(profile).delete_rescue_iso(record["volume_name"]) except Exception: LOG.exception("Could not clean failed rescue ISO import: job=%s", job_id) error += ";宿主机上可能留有救援 ISO 卷,请管理员检查" with rescue_import_lock: rescue_import_jobs[job_id] = { "scope": scope, "status": "failed", "phase": "导入失败", "error": error, } try: save_job("rescue_import_job", job_id, rescue_import_jobs[job_id]) except DatabaseError: LOG.exception("Could not persist rescue import failure: job=%s", job_id) finally: image_path.unlink(missing_ok=True) with rescue_import_lock: pending_rescue_imports.discard(scope) @app.post("/api/v1/rescue-templates", status_code=202) async def upload_rescue_template( request: Request, name: str = Query(min_length=1, max_length=64), _session: Session = Depends(require_admin_write), ): name = validate_template_name(name) profile = current_profile() scope = rescue_scope(profile) with rescue_import_lock: forbid_long_job_during_update() if scope in pending_rescue_imports: raise TemplateError("已有救援 ISO 正在导入,请等待完成", 409) if any(row["name"].casefold() == name.casefold() for row in rescue_catalog.list(scope)): raise TemplateError("当前宿主机已有同名救援模版", 409) pending_rescue_imports.add(scope) image_path: Path | None = None handed_off = False try: declared_size = request.headers.get("content-length") if declared_size is not None: try: if int(declared_size) > MAX_RESCUE_ISO_BYTES: raise TemplateError("救援 ISO 超过 8 GiB 限制", 413) except ValueError as exc: raise TemplateError("上传长度无效", 400) from exc fd, raw_path = tempfile.mkstemp(prefix=".rescue-upload-", suffix=".iso", dir=settings.data_dir) image_path = Path(raw_path) os.fchmod(fd, 0o600) checksum = hashlib.sha256() size = 0 with os.fdopen(fd, "wb") as output: async for chunk in request.stream(): size += len(chunk) if size > MAX_RESCUE_ISO_BYTES: raise TemplateError("救援 ISO 超过 8 GiB 限制", 413) output.write(chunk) checksum.update(chunk) output.flush() os.fsync(output.fileno()) if internal_mode: validation = request.scope.get("engine_upload_validation", {}) if not validation.get("complete") or not validation.get("valid"): raise HTTPException(status_code=401, detail="救援 ISO 摘要与签名不匹配") verify_bootable_iso(image_path) template_id = uuid.uuid4().hex record = { "id": template_id, "name": name, "volume_name": rescue_volume_name(template_id), "size_bytes": size, "sha256": checksum.hexdigest(), } job_id = str(uuid.uuid4()) with rescue_import_lock: rescue_import_jobs[job_id] = { "scope": scope, "status": "queued", "phase": "等待传输到宿主机", } save_job("rescue_import_job", job_id, rescue_import_jobs[job_id]) try: rescue_import_executor.submit( run_rescue_import, job_id, profile, scope, record, image_path, ) except RuntimeError as exc: rescue_import_jobs.pop(job_id, None) delete_job("rescue_import_job", job_id) raise TemplateError("面板正在停止,无法导入救援 ISO", 503) from exc LOG.info("rescue ISO import queued: job=%s admin=%s", job_id, _session.username) handed_off = True # The worker now owns and removes the temporary file. return {"job_id": job_id, "status": "queued"} finally: if not handed_off: if image_path is not None: image_path.unlink(missing_ok=True) with rescue_import_lock: pending_rescue_imports.discard(scope) @app.get("/api/v1/rescue-templates/import-jobs/{job_id}") def rescue_import_job(job_id: str, _session: Session = Depends(require_admin)): scope = rescue_scope(current_profile()) with rescue_import_lock: job = rescue_import_jobs.get(job_id) if job is None or job["scope"] != scope: raise TemplateError("救援模版导入任务不存在", 404) return {key: value for key, value in job.items() if key != "scope"} @app.delete("/api/v1/rescue-templates/{template_id}") def delete_rescue_template(template_id: str, _session: Session = Depends(require_admin_write)): profile = current_profile() scope = rescue_scope(profile) with rescue_lock, rescue_import_lock: forbid_long_job_during_update() if scope in pending_rescue_imports: raise TemplateError("救援 ISO 正在导入,请稍后删除", 409) record = rescue_catalog.get(scope, template_id) states = (data_store.list("vm_rescue") if data_store is not None else test_rescue_states.copy()) for raw in states.values(): try: state_record = json.loads(raw) except (TypeError, ValueError) as exc: raise DatabaseError("虚拟机救援记录损坏") from exc if not isinstance(state_record, dict): raise DatabaseError("虚拟机救援记录损坏") if state_record.get("volume_name") == record["volume_name"]: raise TemplateError("此救援模版正被虚拟机使用,须先退出救援模式", 409) try: new_service(profile).delete_rescue_iso(record["volume_name"]) except VMError as exc: # Only the target volume's precise absence is idempotent. # A missing/misconfigured pool can also be a 404, but must # preserve the catalog entry so the administrator can retry. if (exc.status != 404 or exc.error_code != "rescue_volume_not_found"): raise rescue_catalog.delete(scope, template_id) LOG.info("rescue ISO deleted: id=%s admin=%s", template_id, _session.username) return {"deleted": True} @app.get("/api/v1/vms") def list_vms(_session: Session = Depends(require_approved)): profile = current_profile() vms = new_service(profile).list_vms() if database_mode: account = account_for_session(_session) if not account["admin"]: owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"]) vms = [vm for vm in vms if vm["uuid"] in owned_ids] records = (data_store.list("vm_credential") if data_store is not None else test_credentials.copy()) return {"items": [decorate_vm( vm, parse_credential(records.get(credential_key(profile, vm["uuid"]))), profile, ) for vm in vms]} @app.get("/api/v1/vm-states") def list_vm_states( ids: str | None = Query(None, max_length=4096), _session: Session = Depends(require_approved), ): profile = current_profile() if ids is None: vms = new_service(profile).list_vm_states() if database_mode: account = account_for_session(_session) if not account["admin"]: owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"]) vms = [vm for vm in vms if vm["uuid"] in owned_ids] return {"items": vms} raw_ids = ids.split(",") if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids): raise VMError("虚拟机状态查询数量必须为 1 到 100", 400) canonical_ids: list[str] = [] seen: set[str] = set() for value in raw_ids: try: canonical = str(uuid.UUID(value)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc if canonical not in seen: canonical_ids.append(canonical) seen.add(canonical) if database_mode: account = account_for_session(_session) if not account["admin"]: owned_ids = state.users.list_owned_vm_ids( host_scope(profile), account["id"] ) if any(vm_id not in owned_ids for vm_id in canonical_ids): raise VMError("虚拟机不存在", 404) return {"items": new_service(profile).list_vm_states(canonical_ids)} @app.get("/api/v1/vm-metadata") def list_vm_metadata( ids: str = Query(..., min_length=1, max_length=4096), _session: Session = Depends(require_approved), ): raw_ids = ids.split(",") if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids): raise VMError("虚拟机元数据查询数量必须为 1 到 100", 400) canonical_ids: list[str] = [] seen: set[str] = set() for value in raw_ids: try: canonical = str(uuid.UUID(value)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc if canonical not in seen: canonical_ids.append(canonical) seen.add(canonical) profile = current_profile() if database_mode: account = account_for_session(_session) if not account["admin"]: owned_ids = state.users.list_owned_vm_ids( host_scope(profile), account["id"] ) if any(vm_id not in owned_ids for vm_id in canonical_ids): # Reject the complete request rather than disclosing which # UUID in a mixed-owner batch exists on this host. raise VMError("虚拟机不存在", 404) return {"items": new_service(profile).list_vm_metadata(canonical_ids)} @app.get("/api/v1/vms/{vm_id}/state") def get_vm_state(vm_id: str, _session: Session = Depends(require_approved)): profile = current_profile() require_vm_access(_session, profile, vm_id) return new_service(profile).get_vm_state(vm_id) @app.get("/api/v1/vms/{vm_id}") def get_vm(vm_id: str, _session: Session = Depends(require_approved)): profile = current_profile() require_vm_access(_session, profile, vm_id) vm = new_service(profile).get_vm(vm_id) return decorate_vm(vm, get_credential(profile, vm["uuid"]), profile) @app.get("/api/v1/vms/{vm_id}/firewall") def get_vm_firewall( vm_id: str, _session: Session = Depends(require_approved), ): profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) observed = new_service(profile).get_firewall_policy(canonical_id) record = load_firewall_record(profile, canonical_id) record_free_default = ( record is None and ( observed.get("filter_mode") == "baseline" or is_default_platform_firewall(canonical_id, observed) ) ) if record_free_default: response = { "enabled": False, "revision": 0, "policy_digest": firewall_policy_digest(False, []), "filter_digest": observed.get("filter_digest"), "rules": [], "sync_state": observed.get("sync_state", "active"), } if observed.get("filter_mode") == "managed": response["filter_version"] = FIREWALL_FILTER_COMPILER_VERSION return response applied = bool( record is not None and observed.get("filter_mode") == "managed" and observed.get("sync_state") == "active" and observed.get("filter_digest") == record["filter_digest"] and observed.get("platform_policy_digest") == record["platform_policy_digest"] ) return { "enabled": record["enabled"] if applied else observed.get("enabled"), "revision": record["revision"] if applied else None, "policy_digest": record["policy_digest"] if applied else None, "filter_digest": observed.get("filter_digest"), "rules": record["rules"] if applied else [], "sync_state": "active" if applied else "drifted", } @app.put("/api/v1/vms/{vm_id}/firewall") def put_vm_firewall( vm_id: str, body: FirewallPolicyBody, _session: Session = Depends(require_internal_service_write), ): """Apply one complete typed IPv4 policy; raw nwfilter XML is forbidden.""" forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) vm_scope = host_scope(profile) # These locks protect only the short conflict check. The remote # libvirt call runs without any host-global lock; EngineApplication's # per-VM target lock serializes conflicting mutations for this VM. with reinstall_lock, rescue_lock: if (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError("虚拟机正在重装系统,暂时不能修改防火墙", 409) if ((vm_scope, canonical_id) in pending_rescues or load_rescue_state(profile, canonical_id) is not None): raise VMError("虚拟机处于救援模式或切换中,暂时不能修改防火墙", 409) requested = body.model_dump() requested_rules = requested["rules"] requested_digest = firewall_policy_digest( requested["enabled"], requested_rules) service = new_service(profile) observed = service.get_firewall_policy(canonical_id) try: platform_policy = normalize_platform_policy_snapshot( observed.get("platform_policy_snapshot") or default_platform_policy_snapshot()) except ValueError as exc: raise VMError( "虚拟机平台网络策略证据无效", 409, error_code="firewall_drift_detected", ) from exc current_filter_digest = build_firewall_filter_xml( canonical_id, requested["enabled"], requested_rules, platform_policy_snapshot=platform_policy)[3] if not hmac.compare_digest( requested["policy_digest"], requested_digest): raise VMError( "防火墙策略摘要与规范化规则不一致", 422, error_code="firewall_rule_invalid", ) previous = load_firewall_record(profile, canonical_id) if previous is not None: if not hmac.compare_digest( previous["platform_policy_digest"], platform_policy["digest"]): raise VMError( "虚拟机平台网络策略与引擎记录不一致", 409, error_code="firewall_drift_detected", ) if requested["revision"] < previous["revision"]: raise VMError("防火墙策略版本已更新,请刷新后重试", 409) if (requested["revision"] == previous["revision"] and requested_digest != previous["policy_digest"]): raise VMError("相同防火墙版本对应了不同规则", 409) if requested["revision"] == previous["revision"]: observed_filter_digest = observed.get("filter_digest") if (observed.get("filter_mode") == "managed" and observed.get("sync_state") == "active" and isinstance(observed_filter_digest, str) and hmac.compare_digest( observed_filter_digest, current_filter_digest)): if (previous["filter_version"] != FIREWALL_FILTER_COMPILER_VERSION or not hmac.compare_digest( previous["filter_digest"], current_filter_digest, )): previous = { **previous, "version": 2, "filter_version": FIREWALL_FILTER_COMPILER_VERSION, "filter_digest": current_filter_digest, "platform_policy_snapshot": platform_policy, "platform_policy_digest": platform_policy["digest"], "applied_at": int(time.time()), } save_firewall_record(profile, canonical_id, previous) return { "enabled": previous["enabled"], "revision": previous["revision"], "policy_digest": previous["policy_digest"], "filter_digest": current_filter_digest, "filter_version": FIREWALL_FILTER_COMPILER_VERSION, "rules": previous["rules"], "sync_state": "active", } legacy_compiler_match = ( previous["filter_version"] < FIREWALL_FILTER_COMPILER_VERSION and observed.get("filter_mode") == "managed" and observed.get("sync_state") == "active" and isinstance(observed_filter_digest, str) and hmac.compare_digest( observed_filter_digest, previous["filter_digest"]) ) if not legacy_compiler_match: raise VMError( "防火墙实际状态与已保存版本不一致,请先核查漂移", 409, error_code="firewall_drift_detected", ) result = service.apply_firewall_policy( canonical_id, enabled=requested["enabled"], rules=requested_rules, ) if result.get("policy_digest") != requested_digest: raise VMError( "防火墙策略摘要校验失败,结果待核对", 503, error_code="firewall_result_uncertain", ) if result.get("platform_policy_digest") != platform_policy["digest"]: raise VMError( "平台网络策略摘要校验失败,结果待核对", 503, error_code="firewall_result_uncertain", ) record = { "version": 2, "filter_version": FIREWALL_FILTER_COMPILER_VERSION, "vm_id": canonical_id, "enabled": requested["enabled"], "revision": requested["revision"], "rules": result["rules"], "policy_digest": result["policy_digest"], "filter_digest": result["filter_digest"], "platform_policy_snapshot": platform_policy, "platform_policy_digest": platform_policy["digest"], "applied_at": int(time.time()), } try: save_firewall_record(profile, canonical_id, record) except Exception as exc: LOG.exception( "Firewall applied but local evidence could not be saved: uuid=%s", canonical_id, ) raise VMError( "防火墙已下发但引擎记录未确认,请查询原操作", 503, error_code="firewall_result_uncertain", ) from exc LOG.info( "vm firewall applied: uuid=%s revision=%s enabled=%s actor=%s", canonical_id, body.revision, body.enabled, _session.username, ) return { "enabled": record["enabled"], "revision": record["revision"], "policy_digest": record["policy_digest"], "filter_digest": record["filter_digest"], "filter_version": record["filter_version"], "rules": record["rules"], "sync_state": "active", } def run_snapshot_job(job_id: str, profile: dict) -> None: """Run one host-local snapshot mutation outside the HTTP request.""" job = None try: with snapshot_lock: job = snapshot_jobs[job_id] if job.get("cancelled_before_start") is True: try: finish_cancelled_snapshot_job(job_id, job) except Exception: LOG.exception( "Could not finish cancelled snapshot in runner: %s", job_id, ) snapshot_futures.pop(job_id, None) return if job.get("status") != "queued": # A submitted callable may only cross the libvirt boundary # after this queued -> running transition. Terminal or # otherwise claimed jobs are never replayed here. if job.get("status") in { "failed", "succeeded", "completed", "uncertain"}: pending_snapshot_jobs.discard( (host_scope(profile), job.get("vm_id"))) snapshot_futures.pop(job_id, None) return job["status"] = "running" job["phase"] = { "create": "正在建立在线快照时间点", "restore": "正在构建快照恢复系统盘", "delete": "正在安全删除快照", }[job["action"]] job["started_at"] = int(time.time()) save_job("snapshot_job", job_id, job) except Exception: LOG.exception("Could not start snapshot job safely: %s", job_id) rollback_complete = False try: if job is None: raise KeyError(job_id) record = get_snapshot_record(job["vm_id"], job["snapshot_key"]) if job["action"] == "create": delete_snapshot_record(job["vm_id"], job["snapshot_key"]) else: record["state"] = "ready" save_snapshot_record(record) rollback_complete = True except Exception: LOG.exception("Could not roll back unstarted snapshot job: %s", job_id) with snapshot_lock: current = snapshot_jobs.get(job_id) if current is not None: current.update( status=("failed" if rollback_complete else "uncertain"), phase=("快照任务未启动" if rollback_complete else "快照登记回滚结果待核查"), error=( "快照任务状态无法持久化,尚未调用虚拟化操作" if rollback_complete else "快照任务尚未调用虚拟化操作,但登记状态回滚无法确认" ), finished_at=int(time.time()), ) try: save_job("snapshot_job", job_id, current) except DatabaseError: LOG.exception("Could not persist unstarted snapshot failure: %s", job_id) if isinstance(job, dict) and isinstance(job.get("vm_id"), str): pending_snapshot_jobs.discard( (host_scope(profile), job["vm_id"])) snapshot_futures.pop(job_id, None) return def report_progress(phase: str) -> None: labels = { "freezing": "正在通过 QEMU Guest Agent 冻结访客文件系统", "thawed": "访客文件系统已确认解冻", "copying": "正在复制在线快照数据", "verifying": "正在验证独立快照卷", "cloning": "正在构建快照恢复系统盘", "shutting_down": "正在正常关闭虚拟机以切换系统盘", "switching": "正在切换快照恢复系统盘", "starting": "正在从快照恢复系统盘启动", } label = labels.get(phase) if label is None: return with snapshot_lock: current = snapshot_jobs.get(job_id) if current is not None and current.get("status") == "running": current["phase"] = label if phase == "freezing": current["guest_freeze_state"] = "attempted" elif phase == "thawed": current["guest_freeze_state"] = "thawed" save_job("snapshot_job", job_id, current) action = job["action"] vm_id = job["vm_id"] snapshot_key = job["snapshot_key"] uncertain = False physical_committed = False committed_record = None try: service = new_service(profile) if action == "create": result = service.create_live_snapshot( vm_id, snapshot_key, consistency=job["consistency"], progress=report_progress) physical_committed = True verified_capacity = result.get("size_bytes") verified_identity = result.get("volume_identity") if (isinstance(verified_capacity, bool) or not isinstance(verified_capacity, int) or verified_capacity <= 0 or not isinstance(verified_identity, str) or re.fullmatch(r"[0-9a-f]{64}", verified_identity) is None): raise VMSnapshotUncertainError( "快照复制完成,但卷验证标记无效") with snapshot_lock: current = snapshot_jobs[job_id] current["backup_verified_capacity"] = verified_capacity current["backup_verified_identity"] = verified_identity save_job("snapshot_job", job_id, current) try: service.release_live_snapshot_completed_evidence(vm_id) except Exception: # The durable marker is now authoritative. Failure to # prune retained libvirt stats must not erase success. LOG.exception( "Could not release retained snapshot evidence: job=%s", job_id, ) record = get_snapshot_record(vm_id, snapshot_key) committed_record = record if result.get("volume_name") != record["volume_name"]: raise VMSnapshotUncertainError( "快照复制完成,但存储卷身份与登记记录不一致") now = int(time.time()) record.update( state="ready", completed_at=now, size_bytes=int(result["size_bytes"]), ) save_snapshot_record(record) public_result = { "snapshot_key": snapshot_key, "created": True, "consistency": record["consistency"], "created_at": record["created_at"], "size_bytes": record["size_bytes"], "power_state": result.get("power_state", "running"), } elif action == "restore": record = get_snapshot_record(vm_id, snapshot_key) committed_record = record plan = service.snapshot_restore_plan( vm_id, job["operation_id"]) with snapshot_lock: current = snapshot_jobs[job_id] current["restore_old_volume"] = plan["old_volume_name"] current["restore_replacement_volume"] = ( plan["replacement_volume_name"]) current["restore_was_running"] = plan["was_running"] save_job("snapshot_job", job_id, current) result = service.restore_live_snapshot( vm_id, snapshot_key, record["volume_name"], job["operation_id"], expected_old_volume=plan["old_volume_name"], progress=report_progress) physical_committed = True now = int(time.time()) record.update(state="ready", last_restored_at=now) save_snapshot_record(record) public_result = { "snapshot_key": snapshot_key, "restored": True, "consistency": record["consistency"], "power_state": result.get("power_state", "stopped"), } if isinstance(result.get("warning"), str): public_result["warning"] = result["warning"] else: record = get_snapshot_record(vm_id, snapshot_key) committed_record = record service.delete_snapshot_volume( vm_id, snapshot_key, record["volume_name"]) physical_committed = True # Physical deletion commits before the encrypted registry row # is removed. A registry failure therefore remains uncertain, # never a retryable failed DELETE. delete_snapshot_record(vm_id, snapshot_key) public_result = { "snapshot_key": snapshot_key, "deleted": True, "consistency": record["consistency"], } with snapshot_lock: current = snapshot_jobs[job_id] current.update( status="succeeded", phase={ "create": "在线快照创建完成", "restore": "快照恢复完成", "delete": "快照删除完成", }[action], result=public_result, finished_at=int(time.time()), ) save_job("snapshot_job", job_id, current) LOG.info( "vm snapshot %s succeeded: job=%s uuid=%s snapshot=%s", action, job_id, vm_id, snapshot_key) except Exception as exc: uncertain = (physical_committed or isinstance(exc, VMSnapshotUncertainError)) if not isinstance(exc, (VMError, DatabaseError, ValueError)): uncertain = True LOG.exception( "VM snapshot %s failed unexpectedly: job=%s uuid=%s", action, job_id, vm_id) error = (str(exc) if isinstance(exc, (VMError, DatabaseError, ValueError)) else "快照任务出现未确认错误,请管理员核查引擎日志和宿主机存储") error_code = (exc.error_code if isinstance(exc, VMError) and exc.error_code in SNAPSHOT_PUBLIC_ERROR_CODES else None) try: record = (committed_record if committed_record is not None else get_snapshot_record(vm_id, snapshot_key)) if uncertain: record["state"] = "uncertain" save_snapshot_record(record) elif action == "create": delete_snapshot_record(vm_id, snapshot_key) else: record["state"] = "ready" save_snapshot_record(record) except Exception: uncertain = True LOG.exception("Could not reconcile snapshot record after job failure: %s", job_id) with snapshot_lock: current = snapshot_jobs[job_id] current.update( status="uncertain" if uncertain else "failed", phase="快照结果待核查" if uncertain else "快照任务失败", error=("快照结果不确定,不能重复提交;请管理员核查宿主机存储和虚拟机定义" if uncertain else error), finished_at=int(time.time()), ) if error_code is not None: current["error_code"] = error_code try: save_job("snapshot_job", job_id, current) except DatabaseError: LOG.exception("Could not persist snapshot job failure: %s", job_id) finally: with snapshot_lock: pending_snapshot_jobs.discard((host_scope(profile), vm_id)) snapshot_futures.pop(job_id, None) def reconcile_snapshot_job(job_id: str) -> dict: """Safely converge one uncertain job from read-only host evidence.""" with snapshot_lock: current = snapshot_jobs.get(job_id) if current is None and data_store is not None: raw = data_store.get("snapshot_job", job_id) if raw is not None: try: persisted = json.loads(raw) except (TypeError, ValueError): persisted = None if isinstance(persisted, dict): with snapshot_lock: current = snapshot_jobs.setdefault(job_id, persisted) with snapshot_lock: current = snapshot_jobs.get(job_id) if current is None: raise VMError("快照任务不存在", 404) if current.get("status") != "uncertain": return dict(current) job = dict(current) profile = manager.current() if profile is None or profile.get("key_locked"): return job try: record = None try: record = get_snapshot_record( job["vm_id"], job["snapshot_key"]) except VMError as exc: if job.get("action") != "delete" or exc.status != 404: raise service = new_service(profile) if (job.get("action") == "create" and job.get("consistency") == "filesystem" and job.get("guest_freeze_state") == "attempted"): service.thaw_guest_filesystems(job["vm_id"]) with snapshot_lock: current = snapshot_jobs.get(job_id) if current is None or current.get("status") != "uncertain": return dict(current or job) current["guest_freeze_state"] = "recovered" current["freeze_recovery"] = "confirmed" current["freeze_recovered_at"] = int(time.time()) save_job("snapshot_job", job_id, current) job = dict(current) if job["action"] == "create": if record is None: return job evidence = service.reconcile_live_snapshot_create( job["vm_id"], job["snapshot_key"], record["volume_name"], verified_capacity=job.get("backup_verified_capacity"), verified_identity=job.get("backup_verified_identity"), ) elif job["action"] == "delete": snapshot_volume = ( record["volume_name"] if record is not None else snapshot_volume_name( job["vm_id"], job["snapshot_key"]) ) evidence = service.reconcile_live_snapshot_delete( job["vm_id"], job["snapshot_key"], snapshot_volume) else: if record is None: return job old_volume = job.get("restore_old_volume") was_running = job.get("restore_was_running") replacement_volume = job.get("restore_replacement_volume") if (old_volume is None and was_running is None and replacement_volume is None): # The runner persists all three plan fields atomically # before calling libvirt. Their complete absence proves # restore never crossed its physical mutation boundary. evidence = { "outcome": "failed", "snapshot_key": job["snapshot_key"], } elif (not isinstance(old_volume, str) or not isinstance(was_running, bool) or not isinstance(replacement_volume, str)): return job else: evidence = service.reconcile_live_snapshot_restore( job["vm_id"], job["snapshot_key"], record["volume_name"], job["operation_id"], old_volume, was_running) except (VMError, DatabaseError, KeyError, ValueError): # An unavailable host, damaged evidence, or missing private # restore tombstone is not permission to guess a terminal result. return job outcome = evidence.get("outcome") if outcome not in {"succeeded", "failed"}: return job now = int(time.time()) if outcome == "succeeded": if job["action"] == "create": record.update( state="ready", completed_at=now, size_bytes=int(evidence["size_bytes"]), ) save_snapshot_record(record) public_result = { "snapshot_key": job["snapshot_key"], "created": True, "consistency": record["consistency"], "created_at": record["created_at"], "size_bytes": record["size_bytes"], "power_state": evidence.get("power_state", "stopped"), } elif job["action"] == "restore": record.update(state="ready", last_restored_at=now) save_snapshot_record(record) public_result = { "snapshot_key": job["snapshot_key"], "restored": True, "consistency": record["consistency"], "power_state": evidence.get("power_state", "stopped"), } else: if record is not None: delete_snapshot_record(job["vm_id"], job["snapshot_key"]) public_result = { "snapshot_key": job["snapshot_key"], "deleted": True, "consistency": job["consistency"], } else: if record is None: return job if job["action"] == "create": # Reconciliation proved that no usable recovery point was # created. Preserve the durable job audit, but remove the # hidden catalog row so it cannot accumulate outside the # two-snapshot UI and later block VM expiry deletion. delete_snapshot_record(job["vm_id"], job["snapshot_key"]) else: record["state"] = "ready" save_snapshot_record(record) public_result = None with snapshot_lock: current = snapshot_jobs.get(job_id) if current is None or current.get("status") != "uncertain": return dict(current or job) current.update( status=outcome, phase=("快照自动核对完成" if outcome == "succeeded" else "快照自动核对确认未生效"), finished_at=now, ) if public_result is not None: current["result"] = public_result current.pop("error", None) current.pop("error_code", None) else: current.pop("result", None) current["error"] = "快照操作经宿主机证据确认未生效" current.pop("error_code", None) save_job("snapshot_job", job_id, current) saved = dict(current) if job["action"] == "create": try: service.release_live_snapshot_completed_evidence(job["vm_id"]) except Exception: LOG.exception( "Could not release reconciled snapshot evidence: job=%s", job_id, ) return saved app.state.reconcile_snapshot_job = reconcile_snapshot_job def snapshot_job_state(job_id: str) -> dict | None: with snapshot_lock: current = snapshot_jobs.get(job_id) if (current is not None and current.get("status") == "queued" and current.get("cancelled_before_start") is True): try: current = finish_cancelled_snapshot_job(job_id, current) except Exception: LOG.exception( "Could not finish cancelled snapshot during poll: %s", job_id, ) return dict(current) if current is not None else None app.state.snapshot_job_state = snapshot_job_state def queue_snapshot_job( *, profile: dict, owner: str, vm_id: str, snapshot_key: str, action: Literal["create", "restore", "delete"], consistency: str, ) -> dict: job_id = str(uuid.uuid4()) delegated = TRUSTED_ACTOR.get() if internal_mode else None operation_id = ( delegated.get("operation_id") if isinstance(delegated, dict) and delegated.get("operation_id") else job_id ) job = { "job_id": job_id, "operation_id": operation_id, "scope": host_scope(profile), "owner": owner, "vm_id": vm_id, "snapshot_key": snapshot_key, "action": action, "consistency": consistency, "status": "queued", "phase": "等待快照任务开始", "created_at": int(time.time()), } # Do not publish an in-memory queued job until its durable identity # exists. Otherwise an initial database failure leaves a ghost with no # Future that can permanently block fail-safe network isolation. save_job("snapshot_job", job_id, job) snapshot_jobs[job_id] = job pending_snapshot_jobs.add((job["scope"], vm_id)) try: future = snapshot_executor.submit( run_snapshot_job, job_id, profile) snapshot_futures[job_id] = future except RuntimeError as exc: pending_snapshot_jobs.discard((job["scope"], vm_id)) terminal = { **job, "status": "failed", "phase": "快照任务未进入执行器", "error": "快照任务在开始前失败,未调用虚拟化操作", "failed_before_start": True, "finished_at": int(time.time()), } snapshot_jobs[job_id] = terminal try: save_job("snapshot_job", job_id, terminal) except DatabaseError: # Keep the in-memory terminal proof. Operation polling compares # it with the durable queued row and retries persistence; the # original mutation is never submitted or replayed. LOG.exception( "Could not persist pre-start snapshot submit failure: %s", job_id, ) raise VMError("引擎正在停止,无法开始快照任务", 503) from exc return {"job_id": job_id, "status": "queued"} @app.get("/api/v1/vms/{vm_id}/snapshots") def list_vm_snapshots( vm_id: str, _session: Session = Depends(require_approved), ): profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) return {"items": [ public_snapshot_record(record) for record in list_snapshot_records(canonical_id) ], "limit": 2} @app.get("/api/v1/vms/{vm_id}/snapshots/{snapshot_key}") def get_vm_snapshot( vm_id: str, snapshot_key: str, _session: Session = Depends(require_approved), ): profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) return public_snapshot_record( get_snapshot_record(canonical_id, snapshot_key)) @app.post("/api/v1/vms/{vm_id}/snapshots", status_code=202) def create_vm_snapshot( vm_id: str, body: CreateSnapshotBody, _session: Session = Depends(require_approved_write), ): preflight_profile = current_profile() preflight_id = require_vm_access( _session, preflight_profile, vm_id) forbid_snapshot_suspend_barrier( preflight_profile, preflight_id, "创建快照") with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) scope = host_scope(profile) forbid_pending_snapshot(profile, canonical_id, "创建快照") if ((template_scope(profile), canonical_id) in pending_reinstalls or (scope, canonical_id) in pending_rescues): raise VMError("虚拟机正在执行重装或救援任务,不能创建快照", 409) service = new_service(profile) vm = service.get_vm(canonical_id) if (not vm.get("managed") or vm.get("os_source") != "cloud_template"): raise VMError("只有面板创建的云镜像虚拟机支持本地快照", 409) if (vm.get("rescue_mode") or vm.get("rescue_staged")): raise VMError("虚拟机处于救援模式,不能创建快照", 409) records = list_snapshot_records(canonical_id) if len([row for row in records if row["state"] != "failed"]) >= 2: raise VMError("每台虚拟机最多保留两个本地快照", 409) record = { "snapshot_key": body.snapshot_key, "vm_id": canonical_id, "name": body.name, "state": "creating", "consistency": body.consistency_mode, "volume_name": snapshot_volume_name( canonical_id, body.snapshot_key), "created_at": int(time.time()), } if not insert_snapshot_record(record): raise VMError("快照标识已存在,不能重复创建", 409) try: queued = queue_snapshot_job( profile=profile, owner=_session.username, vm_id=canonical_id, snapshot_key=body.snapshot_key, action="create", consistency=body.consistency_mode) except Exception: delete_snapshot_record(canonical_id, body.snapshot_key) raise return queued @app.post( "/api/v1/vms/{vm_id}/snapshots/{snapshot_key}/restore", status_code=202, ) def restore_vm_snapshot( vm_id: str, snapshot_key: str, _body: EmptySnapshotBody, _session: Session = Depends(require_approved_write), ): preflight_profile = current_profile() preflight_id = require_vm_access( _session, preflight_profile, vm_id) forbid_snapshot_suspend_barrier( preflight_profile, preflight_id, "恢复快照") with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) scope = host_scope(profile) forbid_pending_snapshot(profile, canonical_id, "恢复快照") if ((template_scope(profile), canonical_id) in pending_reinstalls or (scope, canonical_id) in pending_rescues): raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409) record = get_snapshot_record(canonical_id, snapshot_key) if record["state"] != "ready": raise VMError("快照当前不可恢复,请刷新状态后重试", 409) vm = new_service(profile).get_vm(canonical_id) if vm.get("rescue_mode") or vm.get("rescue_staged"): raise VMError("虚拟机处于救援模式,不能恢复快照", 409) record["state"] = "restoring" save_snapshot_record(record) try: queued = queue_snapshot_job( profile=profile, owner=_session.username, vm_id=canonical_id, snapshot_key=snapshot_key, action="restore", consistency=record["consistency"]) except Exception: record["state"] = "ready" save_snapshot_record(record) raise return queued @app.delete( "/api/v1/vms/{vm_id}/snapshots/{snapshot_key}", status_code=202, ) def delete_vm_snapshot( vm_id: str, snapshot_key: str, _body: EmptySnapshotBody, _session: Session = Depends(require_approved_write), ): preflight_profile = current_profile() preflight_id = require_vm_access( _session, preflight_profile, vm_id) forbid_snapshot_suspend_barrier( preflight_profile, preflight_id, "删除快照") with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) scope = host_scope(profile) forbid_pending_snapshot(profile, canonical_id, "删除快照") if ((template_scope(profile), canonical_id) in pending_reinstalls or (scope, canonical_id) in pending_rescues): raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409) record = get_snapshot_record(canonical_id, snapshot_key) if record["state"] != "ready": raise VMError("快照当前不可删除,请刷新状态后重试", 409) record["state"] = "deleting" save_snapshot_record(record) try: queued = queue_snapshot_job( profile=profile, owner=_session.username, vm_id=canonical_id, snapshot_key=snapshot_key, action="delete", consistency=record["consistency"]) except Exception: record["state"] = "ready" save_snapshot_record(record) raise return queued @app.post("/api/v1/vms/{vm_id}/credential") def reveal_vm_credential(vm_id: str, _session: Session = Depends(require_approved_write)): profile = current_profile() require_vm_access(_session, profile, vm_id) vm = new_service(profile).get_vm(vm_id) credential = get_credential(profile, vm["uuid"]) decorated = decorate_vm(vm, credential, profile) LOG.info("vm credential viewed: uuid=%s admin=%s", vm["uuid"], _session.username) return { "login_name": decorated["login_name"], "auth_mode": decorated["auth_mode"], "password": credential["password"] if decorated["password_available"] else None, } @app.get("/api/v1/vms/{vm_id}/metrics") def get_vm_metrics( vm_id: str, range_: Literal["1h", "24h", "7d", "30d"] = Query("1h", alias="range"), _session: Session = Depends(require_approved), ): try: canonical_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc profile = current_profile() require_vm_access(_session, profile, canonical_id) # Never reveal another host's retained samples for a reused UUID. new_service(profile).get_vm(canonical_id) backend = metric_store if metric_store is not None else data_store if backend is None: raise HTTPException(status_code=503, detail="指标数据库尚未就绪") now = int(time.time()) window = RANGES[range_] interval = query_interval_seconds(window) since = max(now - window, now - RETENTION_SECONDS) try: scope = host_scope(profile) except (KeyError, ValueError, TypeError) as exc: raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc points = backend.query_metric_samples( scope, canonical_id, since=since, until=now, interval_seconds=interval, limit=MAX_QUERY_POINTS, ) return {"vm_id": canonical_id, "range": range_, "interval_seconds": interval, "points": points} @app.get("/api/v1/vms/{vm_id}/traffic/daily") def get_vm_daily_traffic( vm_id: str, days: int = Query(30, ge=1, le=30), _session: Session = Depends(require_approved), ): try: canonical_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc profile = current_profile() require_vm_access(_session, profile, canonical_id) # Match the metrics endpoint's live inventory check before consulting # retained rows. The owner and host filters below isolate old tenants. new_service(profile).get_vm(canonical_id) backend = metric_store if metric_store is not None else data_store if backend is None: raise HTTPException(status_code=503, detail="指标数据库尚未就绪") try: scope = host_scope(profile) except (KeyError, ValueError, TypeError) as exc: raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc owner_id = (state.users.owner_of_vm(scope, canonical_id) if database_mode else None) configured_host_id = profile.get("host_id") metric_host_id = configured_host_id if isinstance(configured_host_id, str) else None now = int(time.time()) day_seconds = 24 * 60 * 60 today_start = now - now % day_seconds since = max(0, today_start - (days - 1) * day_seconds) rows = backend.query_daily_traffic( scope, canonical_id, host_id=metric_host_id, user_id=owner_id, since=since, until=now, limit=days, ) by_day = {int(row["day_start"]): row for row in rows} points = [] for day_start in range(since, today_start + 1, day_seconds): row = by_day.get(day_start) points.append({ "date": time.strftime("%Y-%m-%d", time.gmtime(day_start)), "day_start": day_start, "download_bytes": row.get("download_bytes") if row else None, "upload_bytes": row.get("upload_bytes") if row else None, "total_bytes": row.get("total_bytes") if row else None, }) return {"vm_id": canonical_id, "days": days, "timezone": "UTC", "from": since, "until": now, "points": points} @app.post("/api/v1/vms", status_code=201) def create_vm(body: CreateISOBody | CreateCloudBody, _session: Session = Depends(require_approved_write)): # Keep the update submission lock until libvirt has finished creating # the VM. Otherwise an update can restart the web process mid-create. with reinstall_lock: forbid_long_job_during_update() profile = current_profile() account = account_for_session(_session) owner_id = account["id"] if database_mode and (internal_mode or not account["admin"]) else None if isinstance(body, CreateCloudBody): public_key = body.guest_ssh_public_key.strip() root_password = (body.root_password.get_secret_value() if body.root_password is not None else "") if body.guest_ssh_key_id is not None: if public_key: raise HTTPException( status_code=422, detail="已保存公钥和手动公钥只能选择一种", ) saved_key = ssh_key_store().get_for_user( account["id"], body.guest_ssh_key_id, ) if saved_key is None: raise SSHKeyError("公钥不存在", 404) public_key = saved_key["public_key"] # A cloud VM always has a recoverable initial root password, # even when a saved or one-off SSH key was selected. if not root_password: root_password = generate_root_password() password_hash = None if root_password: try: password_hash = hash_root_password(root_password) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc configured_public_pools, _compatibility_mode = public_network_pools( profile) auto_public_ip = bool(configured_public_pools) scheduled_public_ip = bool(body.public_ip_cidr) selected_public_pool = None if auto_public_ip: if body.gateway or body.dns_servers: raise HTTPException( status_code=422, detail="宿主机已配置公网地址池,网关和 DNS 必须使用宿主机配置", ) if scheduled_public_ip and not internal_mode: raise HTTPException( status_code=422, detail="宿主机已配置公网地址池,创建时不能手动指定 IP", ) if scheduled_public_ip and not actor().get("can_provision"): raise HTTPException( status_code=403, detail="只有已确认订单的内部调度任务可以指定预留公网 IP", ) if body.public_pool_key: selected_public_pool = public_pool_for_key( profile, body.public_pool_key, active=True) elif scheduled_public_ip: # Old schedulers did not send a pool key. Infer it # only when the exact reserved address has one # unambiguous active-pool owner. selected_public_pool = public_pool_for_ip( profile, body.public_ip_cidr, active=True) else: # Preserve the legacy automatic-allocation request # during the migration window. Pools are normalized # by key, so every process makes the same choice. selected_public_pool = next( (pool for pool in configured_public_pools if pool["state"] == "active"), None) if selected_public_pool is None: raise HTTPException( status_code=409, detail="宿主机没有开放新地址分配的公网地址池", ) if body.network != selected_public_pool["network_selector"]: raise HTTPException( status_code=422, detail="虚拟机网络与所选公网地址池不一致", ) elif body.public_pool_key: raise HTTPException( status_code=422, detail="宿主机尚未配置可用的公网地址池", ) if not auto_public_ip and not ( body.public_ip_cidr and body.gateway and body.dns_servers ): raise HTTPException( status_code=422, detail="宿主机尚未配置公网地址池,请手动填写 IP、网关和 DNS", ) service = new_service(profile) # The catalog is engine-global. The selected host receives a # verified replica only on first use, before IP/credential/VM # state is mutated. record = ensure_cloud_template_replica( profile, service, body.template_id) spec = body.model_dump(exclude_none=True, exclude={ "source", "template_id", "root_password", "guest_ssh_key_id", "public_pool_key", }) spec["os_name"] = record.get("name") spec["guest_ssh_public_key"] = public_key if password_hash is not None: spec["root_password_hash"] = password_hash requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4()) spec["vm_uuid"] = requested_uuid allocated_ip = "" owner_assigned = False try: if auto_public_ip: assert selected_public_pool is not None if scheduled_public_ip: allocated_ip = claim_scheduled_public_ip( profile, selected_public_pool, body.public_ip_cidr, requested_uuid, ) else: allocated_ip = allocate_public_ip( profile, selected_public_pool, requested_uuid, ) spec["public_ip_cidr"] = allocated_ip spec["public_pool_key"] = selected_public_pool["pool_key"] spec["gateway"] = selected_public_pool["gateway"] spec["dns_servers"] = list( selected_public_pool["dns_servers"]) else: claim_explicit_public_ip(profile, body.public_ip_cidr, requested_uuid) allocated_ip = body.public_ip_cidr if owner_id is not None: state.users.assign_vm(host_scope(profile), requested_uuid, owner_id) owner_assigned = True save_credential(profile, requested_uuid, ("ssh_key_password" if public_key and root_password else "ssh_key" if public_key else "password"), f"panel-seed-{requested_uuid}.iso", root_password or None) vm = service.create_cloud_vm(spec, record["volume_name"]) except Exception as exc: # An incomplete libvirt rollback may have left the guest # behind. Keep its credential in that exceptional case so # the operator can still recover access if it exists. rollback_incomplete = (internal_mode or (isinstance(exc, VMError) and "回滚不完整" in str(exc))) if not rollback_incomplete: try: delete_credential(profile, requested_uuid) except DatabaseError: LOG.exception("Could not remove unused VM credential for %s", requested_uuid) if owner_assigned: try: state.users.delete_vm_owner(host_scope(profile), requested_uuid) except DatabaseError: LOG.exception("Could not remove unused VM owner for %s", requested_uuid) if allocated_ip: try: release_public_ip_claim(profile, allocated_ip, requested_uuid) except DatabaseError: LOG.exception("Could not release unused public IP claim for %s", requested_uuid) raise vm = verified_created_vm(profile, vm, requested_uuid) else: user_iso_record = None if body.user_iso_id is not None: user_scope = user_iso_scope(account["id"]) user_iso_record = user_iso_catalog.get( user_scope, body.user_iso_id) spec = body.model_dump(exclude_none=True, exclude={ "source", "iso", "user_iso_id", }) else: spec = body.model_dump(exclude_none=True, exclude={ "source", "user_iso_id", }) if database_mode: requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4()) spec["vm_uuid"] = requested_uuid if owner_id is not None: state.users.assign_vm(host_scope(profile), requested_uuid, owner_id) try: service = new_service(profile) if user_iso_record is not None: with user_iso_cache_lock: # Re-read both catalog and host marker only after # entering the per-host cache critical section. # Two first-use VM creates therefore cannot both # act on the same stale marker=false observation. user_iso_record = user_iso_catalog.get( user_iso_scope(account["id"]), user_iso_record["id"], ) active_user_iso_records = user_iso_catalog.list( user_iso_scope(account["id"])) active_user_iso_volumes = { row["volume_name"] for row in active_user_iso_records } replica_ready = trusted_user_iso_replica( account["id"], user_iso_record) # A ready marker proves a prior transfer completed. # Keep that durable proof while validating/reusing # the cache; downgrading it before a fallible source # check would strand an already referenced volume # after an engine restart. if not replica_ready: save_user_iso_replica( account["id"], user_iso_record, status="syncing") if shared_user_iso_runtime is not None: with shared_user_iso_runtime.source( account["id"], user_iso_record["id"]) as ( user_iso_record, source_path): service.cache_user_iso( source_path, account["id"], user_iso_record["volume_name"], active_user_iso_volumes, trusted_existing=replica_ready) save_user_iso_replica( account["id"], user_iso_record, status="ready") prune_user_iso_replica_records( account["id"], {row["id"] for row in active_user_iso_records}, ) vm = service.create_user_iso_vm( spec, account["id"], user_iso_record["volume_name"], user_iso_record["id"], user_iso_record["name"], ) else: source_path = user_iso_files.path( account["id"], user_iso_record["id"], expected_size=user_iso_record["size_bytes"]) service.cache_user_iso( source_path, account["id"], user_iso_record["volume_name"], active_user_iso_volumes, trusted_existing=replica_ready) save_user_iso_replica( account["id"], user_iso_record, status="ready") prune_user_iso_replica_records( account["id"], {row["id"] for row in active_user_iso_records}, ) vm = service.create_user_iso_vm( spec, account["id"], user_iso_record["volume_name"], user_iso_record["id"], user_iso_record["name"], ) else: vm = service.create_vm(spec) except Exception as exc: rollback_incomplete = (internal_mode or (isinstance(exc, VMError) and "回滚不完整" in str(exc))) if owner_id is not None and not rollback_incomplete: state.users.delete_vm_owner(host_scope(profile), requested_uuid) raise if database_mode: vm = verified_created_vm(profile, vm, requested_uuid) LOG.info("vm created: uuid=%s admin=%s", vm.get("uuid"), _session.username) credential = get_credential(profile, vm["uuid"]) if isinstance(body, CreateCloudBody) else None return decorate_vm(vm, credential, profile) def run_reinstall( job_id: str, profile: dict, scope: str, vm_id: str, template_volume_name: str, template_name: str | None, ssh_public_key: str, root_password_hash: str | None, generated_password: str | None, credential_password: str | None, ) -> None: vm_reinstalled = False def report_progress(phase: str) -> None: if not isinstance(phase, str): return with reinstall_lock: job = reinstall_jobs.get(job_id) if job is not None and job["status"] == "running": job["phase"] = phase[:200] # ``cleaning_up`` is emitted only after the replacement # domain has started. Persist that boundary before # deleting either old volume, so a process restart or a # later callback failure cannot turn a committed guest # into a retryable/failed reinstall. if phase in {"cleaning_up", "cleanup_pending", "result_uncertain", "completed"}: job["reinstall_committed"] = True try: save_job("reinstall_job", job_id, job) except Exception: # Progress persistence is advisory. The mutation must # continue; the final result or the committed marker # is persisted by the worker when the callback works. LOG.exception("Could not persist reinstall progress: job=%s", job_id) try: with reinstall_lock: reinstall_jobs[job_id]["status"] = "running" reinstall_jobs[job_id]["phase"] = "正在检查虚拟机" save_job("reinstall_job", job_id, reinstall_jobs[job_id]) result = new_service(profile).reinstall_cloud_vm( vm_id, template_volume_name, os_name=template_name, ssh_public_key=ssh_public_key, root_password_hash=root_password_hash, progress=report_progress, ) if not isinstance(result, dict): raise VMError("重装结果无效,无法确认虚拟机状态", 502) credential_generation = result.get("credential_generation") if not isinstance(credential_generation, str) or not credential_generation: raise VMError("重装结果缺少新的系统配置代号,无法保存凭据", 502) result_status = result.get("status") if result_status is not None and not isinstance(result_status, str): raise VMError("重装结果状态无效,无法确认虚拟机状态", 502) if isinstance(result_status, str): result_status = result_status.strip().lower() warning = result.get("warning") error_code = result.get("error_code") if not isinstance(error_code, str): error_code = None result_committed = result.get("reinstall_committed") is True result_cleanup_pending = result.get("cleanup_pending") is True result_has_warning = isinstance(warning, str) and bool(warning.strip()) result_has_error_code = isinstance(error_code, str) and bool(error_code.strip()) # A service implementation must not return an explicit failed # result after crossing the replacement boundary without marking # it. Only an explicit ``reinstall_committed=false`` is proof of # a pre-commit failure; every other returned non-success result # stays uncertain so it cannot authorize a destructive replay. if (result_status == "failed" and result.get("reinstall_committed") is False): raise VMError("重装结果报告失败,未确认新系统已启动", 502) # The service returns only after the replacement definition has # started. Mark that commit before touching the credential store, # so a persistence error cannot be mistaken for a pre-commit # rollback and leave the panel pointing at the old password. vm_reinstalled = True save_credential(profile, vm_id, ("ssh_key_password" if ssh_public_key and credential_password else "ssh_key" if ssh_public_key else "password"), credential_generation, credential_password) success_statuses = {None, "succeeded", "completed", "done"} partial = ( result_status not in success_statuses or result_has_warning or result_cleanup_pending or result_has_error_code or ( result_committed and result_status not in success_statuses ) ) with reinstall_lock: job = reinstall_jobs[job_id] if partial: if error_code not in REINSTALL_PUBLIC_ERROR_CODES: error_code = ( REINSTALL_CLEANUP_PENDING_ERROR_CODE if result.get("cleanup_pending") is True or (isinstance(warning, str) and warning.strip().startswith("旧存储卷")) else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE ) job["status"] = "uncertain" job["phase"] = ( "重装完成,但旧系统盘清理待核对" if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE else "重装结果待核对" ) job.pop("finished_at", None) job["error_code"] = error_code job["error"] = ( "重装已完成,但旧系统盘清理待核对" if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE else "重装结果待核对" ) job["reinstall_committed"] = True if result.get("cleanup_pending") is True: job["cleanup_pending"] = True vm = result.get("vm") if isinstance(vm, dict): # Keep only the bounded observation needed by the # PHP runtime cache; never persist disk source paths. job["vm"] = { key: vm[key] for key in ("uuid", "name", "state") if key in vm } else: job["status"] = "succeeded" job["phase"] = "重装完成" # A progress marker is needed only while the final result # is unresolved. Do not let it make a normal completion # look like a partial commit to the PHP worker. for key in ("reinstall_committed", "cleanup_pending", "error_code", "error", "warning"): job.pop(key, None) if isinstance(warning, str) and warning: job["warning"] = warning if generated_password is not None: # DatabaseStore encrypts this one-time password. Keep it # until the operator explicitly acknowledges saving it. job["generated_password"] = generated_password save_job("reinstall_job", job_id, job) LOG.info("vm reinstall %s: job=%s uuid=%s", "awaits cleanup reconciliation" if partial else "succeeded", job_id, vm_id) except Exception as exc: if isinstance(exc, (VMError, TemplateError, ValueError)): error = str(exc) else: LOG.exception("VM reinstall failed: job=%s uuid=%s", job_id, vm_id) error = "重装系统失败,请查看面板日志" with reinstall_lock: job = reinstall_jobs[job_id] committed = vm_reinstalled or job.get("reinstall_committed") is True if committed: # The replacement is live even when credential/job # persistence failed. Keep the operation uncertain so it # cannot be presented as success or replayed blindly. job["status"] = "uncertain" job["phase"] = "重装结果待核对" job.pop("finished_at", None) job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE job["error"] = "重装结果待核对;面板未能确认新凭据已保存" job["reinstall_committed"] = True job["warning"] = "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码并检查数据库" if generated_password is not None: job["generated_password"] = generated_password else: job["status"] = "failed" job["phase"] = "重装失败" job["error"] = error job.pop("generated_password", None) try: save_job("reinstall_job", job_id, job) except Exception: LOG.exception("Could not persist reinstall status: job=%s", job_id) # A failed terminal write leaves the durable row at its # previous phase. Keep the in-memory guard fail-closed # and make one best-effort retry with an explicit # uncertain state; a later process restart will normalize # the old running row the same way. job["status"] = "uncertain" job["phase"] = "重装结果待核对" job.pop("finished_at", None) if committed: job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE job["error"] = "重装结果待核对;面板未能保存任务状态" else: job.pop("error_code", None) job["error"] = "重装结果待核对;面板未能保存任务状态" job.pop("generated_password", None) try: save_job("reinstall_job", job_id, job) except Exception: LOG.exception("Could not persist uncertain reinstall status: job=%s", job_id) if not committed: LOG.warning("vm reinstall failed: job=%s uuid=%s", job_id, vm_id) finally: with reinstall_lock: job = reinstall_jobs.get(job_id) if job is None or job.get("status") not in { "queued", "running", "uncertain"}: pending_reinstalls.discard((scope, vm_id)) else: pending_reinstalls.add((scope, vm_id)) @app.post("/api/v1/vms/{vm_id}/reinstall", status_code=202) def reinstall_vm(vm_id: str, body: ReinstallBody, _session: Session = Depends(require_approved_write)): ssh_public_key = body.ssh_public_key.strip() supplied_password = (body.root_password.get_secret_value() if body.root_password is not None else "") if body.ssh_key_id is not None: if ssh_public_key: raise HTTPException( status_code=422, detail="已保存公钥和手动公钥只能选择一种", ) account = account_for_session(_session) saved_key = ssh_key_store().get_for_user(account["id"], body.ssh_key_id) if saved_key is None: raise SSHKeyError("公钥不存在", 404) ssh_public_key = saved_key["public_key"] generated_password = None if not supplied_password and not ssh_public_key: generated_password = generate_root_password() supplied_password = generated_password try: if ssh_public_key: ssh_public_key, _unused_hash = _validate_guest_auth(ssh_public_key, None) root_password_hash = (hash_root_password(supplied_password) if supplied_password else None) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() require_vm_access(_session, profile, vm_id) canonical_id = str(uuid.UUID(vm_id)) forbid_retained_snapshot(profile, canonical_id, "重装系统") service = new_service(profile) rescue_record = current_rescue_state(profile, canonical_id, service) rescue_vm = service.get_vm(canonical_id) if ((host_scope(profile), canonical_id) in pending_rescues or rescue_record is not None or rescue_vm.get("rescue_mode") or rescue_vm.get("rescue_staged")): raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) scope = template_scope(profile) vm = service.get_vm(vm_id) if not vm.get("managed") or not vm.get("public_ip_cidr"): raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409) if body.confirmation_name != vm["name"]: raise HTTPException(status_code=422, detail="确认名称与虚拟机名称不一致") normalized_id = vm["uuid"] key = (scope, normalized_id) if key in pending_reinstalls: raise VMError("该虚拟机正在重装系统", 409) if any( job.get("scope") == scope and job.get("vm_id") == normalized_id and job.get("status") in {"queued", "running", "uncertain"} for job in reinstall_jobs.values()): raise VMError("上一次重装结果待核对,请先由管理员核查虚拟机状态", 409) # Finish and verify the target-host replica before scheduling any # task that can shut down the guest or replace its system disk. record = ensure_cloud_template_replica( profile, service, body.template_id) job_id = str(uuid.uuid4()) reinstall_jobs[job_id] = { "owner": _session.username, "scope": scope, "status": "queued", "phase": "等待重装任务开始", "vm_id": normalized_id, "template_id": record["id"], } if generated_password is not None: # The engine record is encrypted at rest by DatabaseStore and # is the only recovery copy if the process exits after the # replacement guest starts but before the final result save. reinstall_jobs[job_id]["generated_password"] = generated_password save_job("reinstall_job", job_id, reinstall_jobs[job_id]) pending_reinstalls.add(key) try: reinstall_executor.submit( run_reinstall, job_id, profile, scope, normalized_id, record["volume_name"], record.get("name"), ssh_public_key, root_password_hash, generated_password, supplied_password or None, ) except RuntimeError as exc: pending_reinstalls.discard(key) reinstall_jobs.pop(job_id, None) delete_job("reinstall_job", job_id) raise VMError("面板正在停止,无法开始重装任务", 503) from exc LOG.info("vm reinstall queued: job=%s uuid=%s admin=%s", job_id, normalized_id, _session.username) return {"job_id": job_id, "status": "queued"} @app.get("/api/v1/reinstall-jobs/{job_id}") def reinstall_job(job_id: str, _session: Session = Depends(require_approved_write)): scope = template_scope(current_profile()) with reinstall_lock: job = reinstall_jobs.get(job_id) if (job is None or job["scope"] != scope or job["owner"] != _session.username): raise VMError("重装任务不存在", 404) status = public_status(job.get("status")) result: dict[str, Any] = {"status": status} for key in ("job_id", "vm_id", "template_id", "operation_id"): identifier = public_identifier(job.get(key), minimum=1) if identifier is not None: result[key] = identifier if (status == "uncertain" and job.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES): result["error_code"] = job["error_code"] result["phase"] = public_phase( job.get("phase"), status, public_reinstall_phases, kind="reinstall") for key in ("created_at", "started_at", "finished_at"): timestamp = public_timestamp(job.get(key)) if timestamp is not None: result[key] = timestamp if "warning" in job: warning = public_warning( job.get("warning"), public_reinstall_warnings, "重装任务已完成,但仍有需要管理员检查的警告") if warning is not None: result["warning"] = warning if "error" in job: error = canonical_job_text( job.get("error"), public_reinstall_errors, "重装系统失败,请查看面板日志") if error is not None: result["error"] = error if status in {"succeeded", "uncertain"} and isinstance(job.get("generated_password"), str): password = job["generated_password"] if 1 <= len(password) <= 128 and re.fullmatch(r"[^\x00-\x1f\x7f]+", password): result["generated_password"] = password return result @app.post("/api/v1/reinstall-jobs/{job_id}/ack-password") def acknowledge_reinstall_password(job_id: str, _session: Session = Depends(require_approved_write)): scope = template_scope(current_profile()) with reinstall_lock: job = reinstall_jobs.get(job_id) if (job is None or job["scope"] != scope or job["owner"] != _session.username): raise VMError("重装任务不存在", 404) if job["status"] not in {"succeeded", "uncertain"}: raise VMError("重装任务尚未完成,不能确认密码", 409) job.pop("generated_password", None) save_job("reinstall_job", job_id, job) return {"acknowledged": True} def run_rescue_job(job_id: str, profile: dict, vm_id: str, operation: Literal["enter", "exit", "force"], network_suspended: bool = False) -> None: vm_scope = host_scope(profile) key = (vm_scope, vm_id) completed = False try: with rescue_lock: job = rescue_jobs[job_id] job["status"] = "running" job["phase"] = ( "正在预挂载救援 ISO 并等待正常关机" if operation == "enter" else "正在强制断电并从救援 ISO 冷启动" if operation == "force" else "正在正常关机并恢复原系统启动配置" ) save_job("rescue_job", job_id, job) state_record = load_rescue_state(profile, vm_id) if state_record is None: raise VMError("救援恢复记录丢失,已停止操作", 409) state_record["status"] = "exiting" if operation == "exit" else "running" save_rescue_state(profile, vm_id, state_record) service = new_service(profile) if operation == "enter": result = service.enter_rescue_vm(vm_id, state_record["volume_name"], state_record["original_xml"]) elif operation == "force": result = service.force_start_staged_rescue_vm( vm_id, state_record["original_xml"]) else: result = service.exit_rescue_vm( vm_id, state_record["original_xml"], network_suspended=network_suspended) completed = True staged = (operation == "enter" and isinstance(result, dict) and result.get("pending_manual_restart") is True) with rescue_lock: if operation in {"enter", "force"}: state_record["status"] = "staged" if staged else "active" save_rescue_state(profile, vm_id, state_record) else: delete_rescue_state(profile, vm_id) job = rescue_jobs[job_id] job["status"] = "completed" job["phase"] = ("救援 ISO 已预挂载,等待手动强制断电后开机" if staged else "救援系统已启动" if operation in {"enter", "force"} else "已退出救援模式,原系统已启动") if staged: job["pending_manual_restart"] = True if isinstance(result, dict) and result.get("warning"): job["warning"] = str(result["warning"]) save_job("rescue_job", job_id, job) LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id) except Exception as exc: if isinstance(exc, (VMError, TemplateError)): error = str(exc) else: LOG.exception("VM rescue %s failed: job=%s uuid=%s", operation, job_id, vm_id) error = "紧急救援操作失败,请查看面板和宿主机日志" with rescue_lock: try: state_record = load_rescue_state(profile, vm_id) if state_record is not None: # If the original inactive definition survived, the # failed job cannot strand an otherwise normal VM in # a database-only "rescue" state. service = new_service(profile) actual_vm = service.get_vm(vm_id) if (not actual_vm.get("rescue_mode") and rescue_xml_matches( service.prepare_rescue_vm( vm_id, state_record["volume_name"]), state_record["original_xml"], )): delete_rescue_state(profile, vm_id) else: state_record["status"] = "failed" save_rescue_state(profile, vm_id, state_record) except (DatabaseError, VMError, TemplateError): LOG.exception("Could not save rescue recovery state: job=%s", job_id) job = rescue_jobs[job_id] job["status"] = "failed" job["phase"] = "救援操作失败" if "正常关机超时" in error or "无法请求虚拟机正常关机" in error: job["error"] = error else: job["error"] = error + ";请确认虚拟机实际运行状态,必要时联系管理员" if completed: job["error"] = "虚拟机操作已完成,但数据库状态保存失败;请管理员检查救援记录" try: save_job("rescue_job", job_id, job) except DatabaseError: LOG.exception("Could not persist rescue job failure: job=%s", job_id) finally: with rescue_lock: pending_rescues.discard(key) @app.get("/api/v1/vms/{vm_id}/rescue") def get_vm_rescue(vm_id: str, _session: Session = Depends(require_approved)): profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) with rescue_lock: service = new_service(profile) vm = service.get_vm(canonical_id) record = current_rescue_state(profile, vm["uuid"], service) staged = bool(vm.get("rescue_staged")) active = bool(vm.get("rescue_mode")) and not staged if (record is not None and (host_scope(profile), vm["uuid"]) not in pending_rescues): if staged and record["status"] in {"queued", "running", "failed"}: record["status"] = "staged" save_rescue_state(profile, vm["uuid"], record) elif active and record["status"] in {"queued", "running", "staged"}: record["status"] = "active" save_rescue_state(profile, vm["uuid"], record) template_name = None template_id = record["template_id"] if record is not None else None if record is not None: try: template_name = rescue_catalog.get( rescue_scope(profile), record["template_id"])["name"] except TemplateError: pass return { "active": active, "staged": staged, "power_state": vm.get("state", "unknown"), "template_id": template_id, "template_name": template_name, "status": record["status"] if record is not None else None, "job_id": record.get("job_id") if record is not None else None, "can_exit": (active or staged) and record is not None, } @app.post("/api/v1/vms/{vm_id}/rescue", status_code=202) def enter_vm_rescue(vm_id: str, body: EnterRescueBody, _session: Session = Depends(require_approved_write)): with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) forbid_pending_snapshot(profile, canonical_id, "进入救援模式") vm_scope = host_scope(profile) key = (vm_scope, canonical_id) if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError("该虚拟机已有救援或重装任务,暂时不能重复操作", 409) service = new_service(profile) vm = service.get_vm(canonical_id) if body.confirmation_name != vm["name"]: raise VMError("确认名称与虚拟机名称不一致", 422) if vm.get("rescue_mode") or vm.get("rescue_staged"): raise VMError("虚拟机已预挂载救援 ISO 或处于救援模式,请先退出", 409) record = rescue_catalog.get(rescue_scope(profile), body.template_id) original_xml = service.prepare_rescue_vm(canonical_id, record["volume_name"]) prior = load_rescue_state(profile, canonical_id) if prior is not None: if not rescue_xml_matches(prior["original_xml"], original_xml): raise VMError("虚拟机仍有未完成的救援记录,请管理员检查后再操作", 409) delete_rescue_state(profile, canonical_id) job_id = str(uuid.uuid4()) state_record = { "template_id": record["id"], "volume_name": record["volume_name"], "original_xml": original_xml, "status": "queued", "job_id": job_id, } save_rescue_state(profile, canonical_id, state_record) rescue_jobs[job_id] = { "scope": vm_scope, "owner": _session.username, "vm_id": canonical_id, "operation": "enter", "status": "queued", "phase": "等待救援任务开始", } try: save_job("rescue_job", job_id, rescue_jobs[job_id]) pending_rescues.add(key) rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "enter") except Exception: pending_rescues.discard(key) rescue_jobs.pop(job_id, None) delete_job("rescue_job", job_id) delete_rescue_state(profile, canonical_id) raise LOG.info("vm rescue enter queued: job=%s uuid=%s user=%s", job_id, canonical_id, _session.username) return {"job_id": job_id, "status": "queued"} @app.post("/api/v1/vms/{vm_id}/rescue/force-start", status_code=202) def force_start_vm_rescue(vm_id: str, body: ForceRescueBody, _session: Session = Depends(require_approved_write)): """Explicit, confirmed power cycle into an already staged rescue ISO.""" with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) forbid_pending_snapshot(profile, canonical_id, "强制启动救援系统") vm_scope = host_scope(profile) key = (vm_scope, canonical_id) if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409) record = load_rescue_state(profile, canonical_id) if record is None: raise VMError("缺少原始启动配置备份,不能安全强制重启进入救援", 409) vm = new_service(profile).get_vm(canonical_id) if body.confirmation_name != vm["name"]: raise VMError("确认名称与虚拟机名称不一致", 422) if not vm.get("rescue_staged"): raise VMError("救援 ISO 尚未预挂载,或虚拟机已经进入救援模式", 409) job_id = str(uuid.uuid4()) rescue_jobs[job_id] = { "scope": vm_scope, "owner": _session.username, "vm_id": canonical_id, "operation": "force", "status": "queued", "phase": "等待手动确认的救援冷启动", } previous_status = record["status"] previous_job_id = record.get("job_id") try: save_job("rescue_job", job_id, rescue_jobs[job_id]) record["status"] = "queued" record["job_id"] = job_id save_rescue_state(profile, canonical_id, record) pending_rescues.add(key) rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "force") except Exception: pending_rescues.discard(key) rescue_jobs.pop(job_id, None) delete_job("rescue_job", job_id) record["status"] = previous_status record["job_id"] = previous_job_id save_rescue_state(profile, canonical_id, record) raise LOG.warning("vm rescue force-start queued: job=%s uuid=%s user=%s", job_id, canonical_id, _session.username) return {"job_id": job_id, "status": "queued"} @app.post("/api/v1/vms/{vm_id}/rescue/exit", status_code=202) def exit_vm_rescue(vm_id: str, body: ExitRescueBody, _session: Session = Depends(require_approved_write)): with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) forbid_pending_snapshot(profile, canonical_id, "退出救援模式") vm_scope = host_scope(profile) key = (vm_scope, canonical_id) if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409) record = load_rescue_state(profile, canonical_id) if record is None: raise VMError("找不到原始启动配置,无法安全退出救援模式", 409) vm = new_service(profile).get_vm(canonical_id) if body.confirmation_name != vm["name"]: raise VMError("确认名称与虚拟机名称不一致", 422) if not (vm.get("rescue_mode") or vm.get("rescue_staged")): raise VMError("虚拟机当前并非救援模式,请管理员核对配置", 409) job_id = str(uuid.uuid4()) rescue_jobs[job_id] = { "scope": vm_scope, "owner": _session.username, "vm_id": canonical_id, "operation": "exit", "status": "queued", "phase": "等待退出救援模式", } save_job("rescue_job", job_id, rescue_jobs[job_id]) record["status"] = "exiting" record["job_id"] = job_id save_rescue_state(profile, canonical_id, record) pending_rescues.add(key) try: rescue_executor.submit( run_rescue_job, job_id, profile, canonical_id, "exit", body.network_suspended) except RuntimeError as exc: pending_rescues.discard(key) record["status"] = "active" save_rescue_state(profile, canonical_id, record) rescue_jobs.pop(job_id, None) delete_job("rescue_job", job_id) raise VMError("面板正在停止,无法退出救援模式", 503) from exc LOG.info("vm rescue exit queued: job=%s uuid=%s user=%s", job_id, canonical_id, _session.username) return {"job_id": job_id, "status": "queued"} @app.get("/api/v1/rescue-jobs/{job_id}") def get_rescue_job(job_id: str, _session: Session = Depends(require_approved)): profile = current_profile() vm_scope = host_scope(profile) with rescue_lock: job = rescue_jobs.get(job_id) if (job is None or job["scope"] != vm_scope or (job["owner"] != _session.username and not account_for_session(_session)["admin"])): raise VMError("救援任务不存在", 404) require_vm_access(_session, profile, job["vm_id"]) status = public_status(job.get("status")) result: dict[str, Any] = {"status": status} for key in ("job_id", "operation", "operation_id"): identifier = public_identifier(job.get(key), minimum=1) if key == "operation": identifier = job.get(key) if job.get(key) in {"enter", "force", "exit"} else None if identifier is not None: result[key] = identifier result["phase"] = public_phase( job.get("phase"), status, public_rescue_phases, kind="rescue") for key in ("created_at", "started_at", "finished_at"): timestamp = public_timestamp(job.get(key)) if timestamp is not None: result[key] = timestamp if isinstance(job.get("pending_manual_restart"), bool): result["pending_manual_restart"] = job["pending_manual_restart"] if "warning" in job: warning = public_warning( job.get("warning"), public_rescue_warnings, "救援任务已完成,但仍有需要管理员检查的警告") if warning is not None: result["warning"] = warning if "error" in job: error = canonical_job_text( job.get("error"), public_rescue_errors, "紧急救援操作失败,请查看面板和宿主机日志") if error is not None: result["error"] = error return result def apply_private_vm_target(vm_id: str, session: Session, method: str, label: str, **options) -> dict: with reinstall_lock, rescue_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(session, profile, vm_id) # Network isolation is a fail-safe policy target and cannot remain # blocked forever by durable snapshot uncertainty. Service stop is # different: an uncertain restore may later reconcile by restoring # its recorded running state, so bypassing it here could power a # purged guest back on without a durable policy tombstone. fail_safe_suspend = method == "network_suspend" barrier_key = (host_scope(profile), canonical_id) barrier_installed = False with snapshot_lock: if fail_safe_suspend: if barrier_key in snapshot_suspend_barriers: raise VMError( "虚拟机正在执行到期暂停策略,暂时不能重复暂停网络", 409, ) snapshot_suspend_barriers.add(barrier_key) barrier_installed = True try: # Only a Future that the executor confirms was still # pending may be cancelled. Its two-phase tombstone and # catalog rollback complete inside this same lock. cancel_queued_snapshot_for_network_suspend( profile, canonical_id) forbid_pending_snapshot( profile, canonical_id, label, allow_uncertain=True, ignore_suspend_barrier=True, ) except Exception: snapshot_suspend_barriers.discard(barrier_key) barrier_installed = False raise else: forbid_pending_snapshot(profile, canonical_id, label) try: if (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError(f"虚拟机正在重装系统,暂时不能执行{label}", 409) if (host_scope(profile), canonical_id) in pending_rescues: raise VMError(f"虚拟机正在切换救援模式,暂时不能执行{label}", 409) vm = getattr(new_service(profile), method)( canonical_id, **options) finally: if barrier_installed: with snapshot_lock: snapshot_suspend_barriers.discard(barrier_key) LOG.info("vm target state applied: uuid=%s target=%s actor=%s", vm.get("uuid"), method, session.username) return vm @app.post("/api/v1/vms/{vm_id}/service-suspend") def service_suspend_vm( vm_id: str, _session: Session = Depends(require_internal_service_write), ): return apply_private_vm_target( vm_id, _session, "service_suspend", "服务暂停") @app.post("/api/v1/vms/{vm_id}/service-resume") def service_resume_vm( vm_id: str, body: ServiceResumeBody, _session: Session = Depends(require_internal_service_write), ): return apply_private_vm_target( vm_id, _session, "service_resume", "服务恢复", start_if_suspended=body.start_if_suspended) @app.post("/api/v1/vms/{vm_id}/network-suspend") def network_suspend_vm( vm_id: str, _session: Session = Depends(require_internal_service_write), ): return apply_private_vm_target( vm_id, _session, "network_suspend", "网络暂停") @app.post("/api/v1/vms/{vm_id}/network-resume") def network_resume_vm( vm_id: str, _session: Session = Depends(require_internal_service_write), ): return apply_private_vm_target( vm_id, _session, "network_resume", "网络恢复") @app.post("/api/v1/vms/{vm_id}/network") def change_network_vm( vm_id: str, body: ChangeNetworkBody, request: Request, _session: Session = Depends(require_internal_service_write), ): """Change one managed cloud VM's public IPv4 on the same host. The engine validates the host pool and holds the new claim before touching libvirt. A 5xx/transport failure deliberately keeps both IP claims for reconciliation; only deterministic validation failures release the new reservation. """ with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() try: normalized_id = str(uuid.UUID(vm_id)) except (TypeError, ValueError, AttributeError) as exc: raise VMError("虚拟机 UUID 无效", 400) from exc require_vm_access(_session, profile, normalized_id) forbid_pending_snapshot(profile, normalized_id, "修改网络") if (template_scope(profile), normalized_id) in pending_reinstalls: raise VMError("虚拟机正在重装系统,暂时不能修改网络", 409) if (host_scope(profile), normalized_id) in pending_rescues: raise VMError("虚拟机正在切换救援模式,请先修改完成后再试", 409) service = new_service(profile) vm = service.get_vm(normalized_id) if (not vm.get("managed") or vm.get("os_source") != "cloud_template" or not vm.get("public_ip_cidr")): raise VMError("只有面板创建的公网云镜像虚拟机支持更换 IP", 409) if not body.confirm_restart: raise VMError("更换公网 IP 前必须确认会短暂重启虚拟机", 422) old_ip = str(vm["public_ip_cidr"]) try: old_pool_key = vm.get("public_pool_key") # Guests created before pool metadata existed, and guests # created while the synthetic legacy pool was active, are # adopted by exact CIDR membership after the native catalog # is installed. if (isinstance(old_pool_key, str) and old_pool_key and old_pool_key != "legacy-default"): current_pool = public_pool_for_key( profile, old_pool_key, active=False) inferred_current_pool = public_pool_for_ip( profile, old_ip, active=False) if inferred_current_pool["pool_key"] != current_pool["pool_key"]: raise VMError( "虚拟机公网 IP 与记录的地址池不一致,已停止网络修改", 409, ) else: current_pool = public_pool_for_ip( profile, old_ip, active=False) target_pool = ( public_pool_for_key(profile, body.public_pool_key, active=True) if body.public_pool_key else public_pool_for_ip(profile, body.public_ip_cidr, active=True) ) except VMError: raise except (TypeError, ValueError) as exc: raise VMError("虚拟机当前公网 IP 配置无效,已停止网络修改", 409) from exc if (target_pool["network_selector"] != current_pool["network_selector"]): raise VMError( "不允许通过换 IP 接口迁移到不同的物理网络接入", 409, ) try: validated = validate_cloud_network( body.public_ip_cidr, target_pool["gateway"], target_pool["dns_servers"], require_auth=False, ) except (TypeError, ValueError) as exc: raise VMError("目标公网 IP 配置无效", 422) from exc if validated["public_ip_cidr"] == old_ip: return { "vm": vm, "public_pool_key": target_pool["pool_key"], "public_ip_cidr": old_ip, "gateway": validated["gateway"], "dns_servers": list(validated["dns_servers"]), "network_changed": False, } upgrade_firewall_filter_if_needed( profile, normalized_id, service) mark_network_mutation_started = request.scope.get( "virt_cloud_mark_network_mutation_started") if callable(mark_network_mutation_started): mark_network_mutation_started() claim_scheduled_public_ip( profile, target_pool, validated["public_ip_cidr"], normalized_id, ) claim_reserved = True try: network = { **validated, "public_pool_key": target_pool["pool_key"], "source_public_pool_key": current_pool["pool_key"], } result = service.change_cloud_vm_network( normalized_id, network, confirm_restart=body.confirm_restart, ) except Exception as exc: # A successful libvirt request can lose its response. Never # release a new claim on a 5xx/unknown result; doing so could # allocate the same address to a second guest. if (claim_reserved and isinstance(exc, VMError) and exc.status < 500): release_public_ip_claim( profile, validated["public_ip_cidr"], normalized_id) raise try: release_public_ip_claim(profile, old_ip, normalized_id) except DatabaseError as exc: LOG.exception("Could not release old public IP claim for %s", normalized_id) # The guest may already use the target address, but PHP must # keep both ledger rows unavailable until the engine claim # swap is complete. A 5xx makes the durable operation # uncertain instead of silently terminal-successful. raise PublicIpClaimCleanupPending( "新公网 IP 已生效,但旧地址占用清理失败,必须人工核查", 503, ) from exc return result @app.post("/api/v1/vms/{vm_id}/settings") def change_vm_settings( vm_id: str, body: VMSettingsBody, _session: Session = Depends(require_approved_write), ): """Change guest hostname/root credentials from the authenticated panel boundary. The public-key value is validated directly from this request. This endpoint never accepts an SSH-key ID and never reads the tenant key catalog. Password hashing and credential persistence happen inside the authenticated engine so plaintext is not written to operation records or logs. """ with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() canonical_id = require_vm_access(_session, profile, vm_id) forbid_pending_snapshot(profile, canonical_id, "修改设置") if (template_scope(profile), canonical_id) in pending_reinstalls: raise VMError("虚拟机正在重装系统,暂时不能修改设置", 409) if (host_scope(profile), canonical_id) in pending_rescues: raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409) hostname = body.hostname if hostname is not None: try: hostname = validate_guest_hostname(hostname) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc public_key = body.ssh_public_key if public_key is not None: try: public_key = _validate_ssh_public_key(public_key.strip()) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc password = (body.root_password.get_secret_value() if body.root_password is not None else None) password_hash = None if password is not None: try: password_hash = hash_root_password(password) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc service = new_service(profile) current_vm = service.get_vm(canonical_id) if (not current_vm.get("managed") or current_vm.get("os_source") != "cloud_template"): raise VMError("只有面板创建的公网云镜像虚拟机支持修改设置", 409) result = service.change_cloud_vm_settings( canonical_id, hostname=hostname, root_password_hash=password_hash, ssh_public_key=public_key, confirm_restart=body.confirm_restart, ) # Keep the panel's encrypted reveal credential in sync only after # the engine confirms the seed was replaced. A missing legacy # record is left untouched rather than inventing an auth mode. existing = get_credential(profile, canonical_id) if password is not None or public_key is not None: if existing is not None: generation = existing["generation"] auth_mode = existing["auth_mode"] saved_password = existing.get("password") else: generation = (result.get("vm", {}).get("credential_generation") if isinstance(result.get("vm"), dict) else None) if not isinstance(generation, str) or not generation: generation = f"panel-seed-{canonical_id}.iso" auth_mode = "ssh_key" if public_key is not None and password is None else "password" saved_password = None if password is not None: saved_password = password auth_mode = "ssh_key_password" if public_key is not None or auth_mode in { "ssh_key", "ssh_key_password" } else "password" elif public_key is not None and auth_mode == "password": auth_mode = "ssh_key_password" save_credential(profile, canonical_id, auth_mode, generation, saved_password) LOG.info("vm settings changed: uuid=%s admin=%s", canonical_id, _session.username) return result @app.post("/api/v1/vms/{vm_id}/actions") def action(vm_id: str, body: ActionBody, _session: Session = Depends(require_approved_write)): with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() require_vm_access(_session, profile, vm_id) try: normalized_id = str(uuid.UUID(vm_id)) except ValueError: normalized_id = vm_id forbid_pending_snapshot(profile, normalized_id, "执行电源操作") if (template_scope(profile), normalized_id) in pending_reinstalls: raise VMError("虚拟机正在重装系统,暂时不能执行其他操作", 409) service = new_service(profile) if (host_scope(profile), normalized_id) in pending_rescues: raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409) rescue_record = current_rescue_state(profile, normalized_id, service) current_vm = service.get_vm(normalized_id) staged = bool(current_vm.get("rescue_staged")) if staged and rescue_record is not None: if body.action not in {"poweroff", "start"}: raise VMError( "救援 ISO 已预挂载;请强制断电后开机进入救援,或取消救援配置", 409) elif rescue_record is not None or current_vm.get("rescue_mode") or staged: raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) vm = service.action(vm_id, body.action) if staged and body.action == "start": if not vm.get("rescue_mode") or vm.get("rescue_staged"): raise VMError( "虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502) rescue_record["status"] = "active" save_rescue_state(profile, normalized_id, rescue_record) LOG.info("vm action: uuid=%s action=%s admin=%s", vm.get("uuid"), body.action, _session.username) return vm @app.delete("/api/v1/vms/{vm_id}") def delete_vm( vm_id: str, delete_disks: bool = False, _session: Session = Depends(require_approved_write), ): with reinstall_lock, rescue_lock, snapshot_lock: forbid_long_job_during_update() profile = current_profile() require_vm_access(_session, profile, vm_id) system_cleanup = ( internal_mode and actor().get("system_cleanup") is True ) try: normalized_id = str(uuid.UUID(vm_id)) except ValueError: normalized_id = vm_id if (host_scope(profile), normalized_id) in pending_snapshot_jobs: raise VMError("虚拟机正在执行快照任务,暂时不能删除", 409) if (template_scope(profile), normalized_id) in pending_reinstalls: raise VMError("虚拟机正在重装系统,暂时不能删除", 409) service = new_service(profile) if (host_scope(profile), normalized_id) in pending_rescues: raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) if not system_cleanup: rescue_record = current_rescue_state(profile, normalized_id, service) rescue_vm = service.get_vm(normalized_id) rescue_protected = ( rescue_record is not None or rescue_vm.get("rescue_mode") or rescue_vm.get("rescue_staged") ) else: rescue_protected = False if rescue_protected: raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) registered_snapshots = list_snapshot_records(normalized_id) if registered_snapshots and not delete_disks: raise VMError("虚拟机仍有本地快照;删除磁盘时会一并清理快照", 409) # Validate every registered recovery point before the VM delete, # but keep all of them intact until that deletion commits. This # prevents a later VM-delete failure from destroying the user's # last rollback points. if delete_disks and registered_snapshots: for snapshot in registered_snapshots: service.validate_snapshot_volume( normalized_id, snapshot["snapshot_key"], snapshot["volume_name"], ) result = service.delete_vm(vm_id, delete_disks) deleted_snapshots = [] if delete_disks and registered_snapshots: try: for snapshot in registered_snapshots: service.delete_snapshot_volume( normalized_id, snapshot["snapshot_key"], snapshot["volume_name"], allow_missing_vm=True, ) delete_snapshot_record( normalized_id, snapshot["snapshot_key"]) deleted_snapshots.append(snapshot["snapshot_key"]) except Exception as exc: # The VM deletion already committed. Keep every # unconfirmed catalog row so reconciliation can identify # exactly which volume still needs attention. raise VMDeleteSnapshotCleanupPending( "虚拟机已删除,但本地快照清理结果不完整;必须核查原删除操作" ) from exc if deleted_snapshots: result["deleted_snapshots"] = deleted_snapshots if system_cleanup: try: delete_rescue_state(profile, result["uuid"]) except Exception as exc: LOG.exception("Could not remove rescue state for deleted VM %s", result["uuid"]) raise VMDeleteSnapshotCleanupPending( "虚拟机已删除,但救援状态清理未完成;必须核查原删除操作" ) from exc if database_mode: try: state.users.delete_vm_owner(host_scope(profile), result["uuid"]) except Exception as exc: LOG.exception("Could not remove owner for deleted VM %s", result["uuid"]) raise VMDeleteSnapshotCleanupPending( "虚拟机已删除,但归属记录清理未完成;必须核查原删除操作" ) from exc try: delete_credential(profile, result["uuid"]) except Exception as exc: LOG.exception("Could not remove credential for deleted VM %s", result["uuid"]) raise VMDeleteSnapshotCleanupPending( "虚拟机已删除,但登录凭据清理未完成;必须核查原删除操作" ) from exc try: release_vm_public_ip_claims(profile, result["uuid"]) except Exception as exc: LOG.exception("Could not release public IP claim for deleted VM %s", result["uuid"]) # Do not let PHP free the assigned address while the engine # still has a stale claim. The deletion outcome stays # uncertain and requires explicit reconciliation. raise PublicIpClaimCleanupPending( "虚拟机已删除,但公网 IP 占用清理失败,必须人工核查", 503, ) from exc try: service.delete_firewall_filter(result["uuid"]) delete_firewall_record(profile, result["uuid"]) except Exception as exc: LOG.exception( "Could not remove firewall state for deleted VM %s", result["uuid"], ) raise VMDeleteFirewallCleanupPending( "虚拟机已删除,但防火墙过滤器清理未完成;必须核查原删除操作" ) from exc LOG.info( "vm deleted: uuid=%s delete_disks=%s admin=%s", result.get("uuid"), delete_disks, _session.username, ) return result @app.websocket("/api/v1/vms/{vm_id}/console") async def console(websocket: WebSocket, vm_id: str): host = websocket.headers.get("host") if websocket.url.scheme != "wss": await websocket.close(code=4403) return if not _origin_matches_host(websocket.headers.get("origin"), host): await websocket.close(code=4403) return session = await asyncio.to_thread(sessions.get, websocket.cookies.get(COOKIE_NAME)) if session is None: await websocket.close(code=4401) return try: account = await asyncio.to_thread(account_for_session, session) except HTTPException: await websocket.close(code=4401) return if not account["admin"] and not account["approved"]: await websocket.close(code=4403) return profile = manager.current() if profile is None: await websocket.close(code=4409) return try: if profile.get("key_locked"): await websocket.close(code=4423) return await asyncio.to_thread(require_vm_access, session, profile, vm_id) port = await asyncio.to_thread(new_service(profile).console_port, vm_id) except VMError: await websocket.close(code=4404) return await websocket.accept() async def watch_console_access() -> None: while True: await asyncio.sleep(CONSOLE_ACCESS_CHECK_SECONDS) try: refreshed = await asyncio.to_thread(sessions.get, session.token) if refreshed is None or refreshed.username != session.username: return account = await asyncio.to_thread(account_for_session, refreshed) if not account["admin"] and not account["approved"]: return current = manager.current() if (current is None or current.get("key_locked") or host_scope(current) != host_scope(profile)): return await asyncio.to_thread(require_vm_access, refreshed, current, vm_id) except (HTTPException, VMError, DatabaseError, RemoteHostError, KeyError, TypeError, ValueError): return relay_task = asyncio.create_task(relay_console( websocket, port=port, ssh_target=profile["ssh_target"], ssh_binary=settings.ssh_binary, known_hosts_file=profile["ssh_known_hosts_file"], ssh_config_file=profile["ssh_config_file"], )) access_task = asyncio.create_task(watch_console_access()) try: done, _pending = await asyncio.wait( {relay_task, access_task}, return_when=asyncio.FIRST_COMPLETED) if access_task in done: relay_task.cancel() await asyncio.gather(relay_task, return_exceptions=True) with suppress(RuntimeError, OSError): await websocket.close(code=4403) return await relay_task except ConsoleRelayError as exc: LOG.warning("VNC relay failed for vm %r: %s", vm_id, exc) with suppress(RuntimeError, OSError): await websocket.close(code=1011) finally: access_task.cancel() await asyncio.gather(access_task, return_exceptions=True) return app